MapReduceBasic/App.java at master · PrashantGupta17/MapReduceBasic · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
packagecom.test.begin;

importjava.io.IOException;

importorg.apache.hadoop.conf.Configuration;
importorg.apache.hadoop.fs.Path;
importorg.apache.hadoop.io.IntWritable;
importorg.apache.hadoop.io.Text;
importorg.apache.hadoop.mapreduce.Job;
importorg.apache.hadoop.mapreduce.Mapper;
importorg.apache.hadoop.mapreduce.Reducer;
importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;
importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * Test Map -Reduce
 *
 */
publicclassApp{

publicstaticclassTestMapextendsMapper<Object, Text, Text, IntWritable>{
//The output value of mapper is kept 1 of type IntWritable, as no specific Object value of use
publicfinalstaticIntWritableoccurence = newIntWritable(1);

publicTextword = newText();

@Override
protectedvoidmap(Objectkey, Textvalue,
Mapper<Object, Text, Text, IntWritable>.Contextcontext)
throwsIOException, InterruptedException {
// TODO Auto-generated method stub
//super.map(key, value, context);
// Under this function you can read each seperate line as VALUE, and dig out the information you ned to pass as key to reducer
// if you know delimeter, you can easily extract tokens by value.toString().split("ddelimeter")
//or you can use StringTokenizer to iterate through the tokens
// you can select any number of key,value pairs to output

// Assuming that I use AllstarFull.csv where records are seperated by ,

String[] words = value.toString().split(",");
word.set(words[0]);
context.write(word, occurence);

 }

// SO, TestMap should now return key value pairs after reading the AllstarFull.csv like(playerId, 1)
// and its gonna read each line and output one key,value pair per line

	}

publicstaticclassTestReduceextendsReducer<Text, IntWritable, Text, IntWritable>{

@Override
protectedvoidreduce(Textarg0, Iterable<IntWritable> arg1,
Reducer<Text, IntWritable, Text, IntWritable>.Contextarg2)
throwsIOException, InterruptedException {
// TODO Auto-generated method stub
//super.reduce(arg0, arg1, arg2);

intsum = 0;
for(IntWritablevalue : arg1){

sum += value.get();

 }

arg2.write(arg0, newIntWritable(sum));
 }


	}

publicstaticvoidmain( String[] args ) throwsException
 {


Configurationconf = newConfiguration();

Jobjob = Job.getInstance(conf, "player count");

job.setJarByClass(App.class);

job.setMapperClass(TestMap.class);

job.setCombinerClass(TestReduce.class);

job.setReducerClass(TestReduce.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(job, newPath(args[0]));

FileOutputFormat.setOutputPath(job, newPath(args[1]));

System.exit(job.waitForCompletion(true)?0:1);
 }
}