《大数据技术原理与应用》实验一WordCount

实验一MapReduce--1--入门程序WordCount一、实验目的MapReduce界的helloworld程序就是WordCount程序。所谓WordCount，就是单词计数，就是用来统计一篇或者一堆文本文件中的各单词的出现次数。二、实验原理按照我们普通的编写代码的逻辑，单词计数程序应该是这样的：1、逐行读取文本内容2、把读取到的一行文本内容切割为一个一个的单词3、把每个单词出现一次的信息记录为一个key-value，也就是“单词-1”4、收集所有相同的单词，然后统计value写出的value值得总和，也就是key为同一个单词的所有1的和。实验过程首先在HDFS上创建输入文件目录test，命令如下：hadoop@ubuntu:~$hadoopfs-mkdirtest接着将本地的文件put到HDFS上test目录下，并查看，命令如下：hadoop@ubuntu:~/txtfile$hadoopfs-put-f*.txttesthadoop@ubuntu:~$hadoopfs-lstestFound3items-rw-r--r--

1hadoopsupergroup

242016-10-2221:43test/file1.txt-rw-r--r--

1hadoopsupergroup

242016-10-2221:43test/file2.txt-rw-r--r--

1hadoopsupergroup

1149572016-10-2319:57test/news.txt注释:-f是可以overwrite的意思使用InteliJ新建一个Javaproject,编写wordcount程序,如下所示：packageexample;importorg.apache.hadoop.fs.Path;importorg.apache.hadoop.io.IntWritable;importorg.apache.hadoop.io.LongWritable;importorg.apache.hadoop.io.Text;importorg.apache.hadoop.mapred.*;importjava.io.IOException;importjava.util.Iterator;importjava.util.StringTokenizer;/**

*Createdbyhadoopon10/22/16.

*/publicclassWordCount{

///Mapper:<LongWritableText>projectto<TextIntWritable>

publicstaticclassMapextendsMapReduceBaseimplementsMapper<LongWritable,Text,Text,IntWritable>{

privatefinalstaticIntWritableone=newIntWritable(1);

privateTextword=newText();

///reporterreporttheprogressortheyarelive

publicvoidmap(LongWritablekey,Textvalue,OutputCollector<Text,IntWritable>output,Reporterreporter)throwsIOException{

Stringline=value.toString();

StringTokenizertokenizer=newStringTokenizer(line);

while(tokenizer.hasMoreTokens()){

//setchangestringtoText

word.set(tokenizer.nextToken());

output.collect(word,one);}}}

publicstaticclassReduceextendsMapReduceBaseimplementsReducer<Text,IntWritable,Text,IntWritable>{

///iscalledforeverykey

///IntWritable:youcanuseitasint

publicvoidreduce(Textkey,Iterator<IntWritable>values,OutputCollector<Text,IntWritable>output,Reporterreporter)throwsIOException{

intsum=0;

while(values.hasNext()){

sum+=values.next().get();

}

output.collect(key,newIntWritable(sum));

}

}

publicstaticvoidmain(String[]args)throwsException{

JobConfconf=newJobConf(WordCount.class);

conf.setJobName("wordcount");

conf.setOutputKeyClass(Text.class);

conf.setOutputValueClass(IntWritable.class);

conf.setMapperClass(Map.class);

//conf.setCombinerClass(Reduce.class);

conf.setReducerClass(Reduce.class);

//inputformat

conf.setInputFormat(TextInputFormat.class);

conf.setOutputFormat(TextOutputFormat.class);

FileInputFormat.setInputPaths(conf,newPath(args[0]));

FileOutputFormat.setOutputPath(conf,newPath(args[1]));

JobClient.runJob(conf);

}}导出jar包File->ProjectStructure->Artifacts->绿色“+”->JAR->Frommoduleswithdependcies->applyBuild->BuildArtifacts运行jar包hadoop@ubuntu:~/txtfile$hadoopjarWordCount.jartesttest-out最后查看HDFS上的结果并将结果get到本地hadoop@ubuntu:~$hadoopfs-lstest-outFound2items-rw-r--r--

1hadoopsupergroup

02016-10-2300:59test-out/_SUCCESS-rw-r--r--

1hadoopsupergroup

402016-10-2300:59te