Java編寫Mapreduce程序過程淺析

第Java編寫Mapreduce程序過程淺析目錄Maven項(xiàng)目下所需依賴數(shù)據(jù)類型一、Mapper類二、Reducer類三、執(zhí)行類程序執(zhí)行結(jié)果一個(gè)Maprduce程序主要包括三部分：Mapper類、Reducer類、執(zhí)行類。

Maven項(xiàng)目下所需依賴

dependencies

dependency

groupIdorg.apache.hadoop/groupId

artifactIdhadoop-client/artifactId

version3.3.0/version

/dependency

dependency

groupIdjunit/groupId

artifactIdjunit/artifactId

version4.12/version

scopetest/scope

/dependency

dependency

groupIdorg.slf4j/groupId

artifactIdslf4j-api/artifactId

version1.7.30/version

/dependency

dependency

groupIdjunit/groupId

artifactIdjunit/artifactId

version3.8.2/version

/dependency

dependency

groupIdjunit/groupId

artifactIdjunit/artifactId

version4.12/version

scopecompile/scope

/dependency

/dependencies

數(shù)據(jù)類型

Text：一種可變長(zhǎng)度的字節(jié)數(shù)組，用于表示文本數(shù)據(jù)。相當(dāng)于Java中的String。LongWritable、IntWritable、FloatWritable、DoubleWritable：分別用于表示長(zhǎng)整型、整型、浮點(diǎn)型和雙精度浮點(diǎn)型數(shù)據(jù)。相當(dāng)于Java中的long、int、float和double。BooleanWritable：用于表示布爾類型數(shù)據(jù)。相當(dāng)于Java中的boolean。NullWritable：用于表示空值，通常用于表示Map任務(wù)的輸出中間結(jié)果數(shù)據(jù)中的值為空。相當(dāng)于Java中的null。ArrayWritable：用于表示數(shù)組類型數(shù)據(jù)。相當(dāng)于Java中的數(shù)組。MapWritable：一種可序列化的Map數(shù)據(jù)結(jié)構(gòu)，可以作為Map任務(wù)的輸出（中間結(jié)果數(shù)據(jù)）或Reduce任務(wù)的輸入。相當(dāng)于Java中的Map。WritableComparable：一種可序列化的、可比較的數(shù)據(jù)類型接口，可以作為Map任務(wù)或Reduce任務(wù)的輸入輸出數(shù)據(jù)類型。

一、Mapper類

importorg.apache.hadoop.io.LongWritable;

importorg.apache.hadoop.io.Text;

importorg.apache.hadoop.mapreduce.Mapper;

importorg.apache.hadoop.util.StringUtils;

importjava.io.IOException;

publicclassWordCountMapperextendsMapperLongWritable,Text,Text,LongWritable{

@Override

protectedvoidmap(LongWritablekey,Textvalue,Contextcontext)throwsIOException,InterruptedException{

//如果當(dāng)前數(shù)據(jù)不為空

if(value!=null){

//獲取每一行的數(shù)據(jù)

Stringline=value.toString();

//將一行數(shù)據(jù)根據(jù)空格分開

//String[]words=line.split("");

String[]words=StringUtils.split(line,'');//hadoop的StringUtils.split方法對(duì)大數(shù)據(jù)來說比Java自帶的擁有更好的性能

//輸出鍵值對(duì)

for(Stringword:words){

context.write(newText(word),newLongWritable(1));

}

二、Reducer類

importorg.apache.hadoop.io.LongWritable;

importorg.apache.hadoop.io.Text;

importorg.apache.hadoop.mapreduce.Reducer;

importjava.io.IOException;

publicclassWordCountReducerextendsReducerText,LongWritable,Text,LongWritable{

@Override

protectedvoidreduce(Textkey,IterableLongWritablevalues,Contextcontext)throwsIOException,InterruptedException{

//累加單詞的數(shù)量

longsum=0;

//遍歷單詞計(jì)數(shù)數(shù)組,將值累加到sum中

for(LongWritablevalue:values){

sum+=value.get();

//輸出每次最終的計(jì)數(shù)結(jié)果

context.write(key,newLongWritable(sum));

}

三、執(zhí)行類

importorg.apache.hadoop.conf.Configuration;

importorg.apache.hadoop.conf.Configured;

importorg.apache.hadoop.fs.Path;

importorg.apache.hadoop.io.LongWritable;

importorg.apache.hadoop.io.Text;

importorg.apache.hadoop.mapreduce.Job;

importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;

importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

importorg.apache.hadoop.util.Tool;

importorg.apache.hadoop.util.ToolRunner;

publicclassWordCountRunnerextendsConfiguredimplementsTool{

publicstaticvoidmain(String[]args)throwsException{

ToolRunner.run(newConfiguration(),newWordCountRunner(),args);

@Override

publicintrun(String[]args)throwsException{

Configurationconf=newConfiguration();

Jobjob=Job.getInstance(conf,"wordcount");

job.setJarByClass(WordCountRunner.class);

job.setMapperClass(WordCountMapper.class);

job.setReducerClass(WordCountReducer.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(LongWritable.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(LongWritable.class);

//設(shè)置統(tǒng)計(jì)文件輸入的路徑,將命令行的第一個(gè)參數(shù)作為輸入文件的路徑

//讀取maven項(xiàng)目下resources目錄的文檔

Stringpath=getClass().getResource("/words.txt").getPath();

FileInputFormat.setInputPaths(job,path);

//設(shè)置結(jié)果數(shù)據(jù)存放路徑,將命令行的第二個(gè)參數(shù)作為數(shù)據(jù)的輸出路徑

//輸出目錄必須不存在!!!

FileOutputFormat.setOutputPath(job,newPath("./output"));

returnjob.waitForCompletion(true)0:1;

}

程序執(zhí)行結(jié)果

.part-r-00000.crc：是Reduce任務(wù)輸出結(jié)果文件的校驗(yàn)文件，用于校驗(yàn)Reduce任務(wù)輸出結(jié)果文件的完整性和正確性。該文件由Hadoop框架自動(dòng)生成，不需要手動(dòng)創(chuàng)建，其內(nèi)容是Reduce任務(wù)輸出結(jié)果文件的校驗(yàn)和信息。._S