public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1); private Text word = new Text(); /* * LongWritable 为输入的key的类型 * Text 为输入value的类型 * Text-IntWritable 为输出key-value键值对的类型 */ public void map(Object key, Text value, Context context ) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString()); // 将TextInputFormat生成的键值对转换成字符串类型 while (itr.hasMoreTokens()) { word.set(itr.nextToken()); context.write(word, one); } } }
public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable> { private IntWritable result = new IntWritable(); /* * Text-IntWritable 来自map的输入key-value键值对的类型 * Text-IntWritable 输出key-value 单词-词频键值对 */ public void reduce(Text key, Iterable<IntWritable> values, Context context ) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } result.set(sum); context.write(key, result); } }
... 省略部分 File System Counters FILE: Number of bytes read=2025025 FILE: Number of bytes written=4443318 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=10356334 HDFS: Number of bytes written=616286 HDFS: Number of read operations=25 HDFS: Number of large read operations=0 HDFS: Number of write operations=5 Map-Reduce Framework Map input records=33907 Map output records=663964 Map output bytes=6687108 Map output materialized bytes=1005779 Input split bytes=216 Combine input records=663964 Combine output records=68147 Reduce input groups=55800 Reduce shuffle bytes=1005779 Reduce input records=68147 Reduce output records=55800 Spilled Records=136294 Shuffled Maps =2 Failed Shuffles=0 Merged Map outputs=2 GC time elapsed (ms)=187 Total committed heap usage (bytes)=1323827200 Shuffle Errors BAD_ID=0 CONNECTION=0 IO_ERROR=0 WRONG_LENGTH=0 WRONG_MAP=0 WRONG_REDUCE=0 File Input Format Counters Bytes Read=4054789 File Output Format Counters Bytes Written=616286
