If you have tried Hadoop WordCount sample job available in multiple old tutorials, you may have hit compilation problem as below:
Older Code:
package org.myorg;
import java.io.Exception;
import java.util.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.*;
public class WordCount {
public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
output.collect(word, one);
}
}
}
public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
int sum = 0;
while (values.hasNext()) {
sum += values.next().get();
}
output.collect(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws Exception {
JobConf conf = new JobConf(WordCount.class);
conf.setJobName(“wordcount”);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(MapClass.class);
conf.setCombinerClass(Reduce.class);
conf.setReducerClass(Reduce.class);
conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
conf.setInputPath(new Path(args[1]));
conf.setOutputPath(new Path(args[2]));
JobClient.runJob(conf);
}
}
After Compilation we hit the following error:
WordCount.java:2: error: cannot find symbol
import java.io.Exception;
^
symbol: class Exception
location: package java.io
WordCount.java:14: error: cannot find symbol
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
^
symbol: class IOException
location: class MapClass
WordCount.java:25: error: cannot find symbol
public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
^
symbol: class IOException
location: class Reduce
WordCount.java:44: error: cannot find symbol
conf.setInputPath(new Path(args[1]));
^
symbol: method setInputPath(Path)
location: variable conf of type JobConf
WordCount.java:45: error: cannot find symbol
conf.setOutputPath(new Path(args[2]));
^
symbol: method setOutputPath(Path)
location: variable conf of type JobConf
Note: WordCount.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
5 errors
This is because the old sample code is based on older Hadoop distribution. This problem happens when you use 0.20.x or newer Hadoop distribution. Like in my case, I was using 0.20.203.1 as below:
C:AzureJava>C:Appsjavaopenjdk7binjavac -classpath c:Appsdisthadoop-core-0.20.203.1-SNAPSHOT.jar -d . WordCount.java
To solve this problem you would need to change your code to as below:
package org.myorg;
import java.io.IOException;
import java.util.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class AvkashWordCount {
public static class Map extends Mapper
<LongWritable, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
context.write(word, one);
}
}
}
public static class Reduce extends Reducer
<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterator<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum = 0;
while (values.hasNext()) {
sum += values.next().get();
}
context.write(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf);
job.setJarByClass(AvkashWordCount.class);
job.setJobName(“avkashwordcountjob”);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(AvkashWordCount.Map.class);
job.setCombinerClass(AvkashWordCount.Reduce.class);
job.setReducerClass(AvkashWordCount.Reduce.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}
}
Above code is tested with Hadoop 0.20.x and above distribution.
Resources:
- http://stackoverflow.com/questions/7626077/mapreducebase-and-mapper-deprecated
- http://marcellodesales.wordpress.com/2010/01/05/hadoop-0-20-1-api-refactoring-the-invertedline-cloudera-trainin-removing-deprecated-classes-jobconf/
- http://sonerbalkir.blogspot.com/2010/01/new-hadoop-api-020x.html
