hadoop学习；block数据块；mapreduce实现样例；UnsupportedClassVersionError异常；关联项目源代码...

对于开源的东东&＃xff0c;尤其是刚出来不久&＃xff0c;我认为最好的学习方式就是能够看源代码和doc&＃xff0c;測试它的样例

为了方便查看源代码&＃xff0c;关联导入源代码的项目

先前的项目导入源代码是关联了源代码文件

block数据块&＃xff0c;在配置文件hdfs-default.xml中能够查看到,记住要改动不是在这里

block文件存储块是最主要的单位

查看block存放位置&＃xff0c;配置文件里查看

假设文件大于64M会占两个块&＃xff0c;meta文件是校验文件&＃xff0c;第二个文件大于64M&＃xff0c;删除文件后&＃xff0c;则相应block不在

datanode存放文件&＃xff0c;一个文件能够存放在不同机器上datanode

mapreduce本身有默认的类&＃xff0c;当什么都不写的时候&＃xff0c;原样输出

package com.kane.mr.minidefault;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class TestDefault {
public static void main(String[] args) throws Exception {
Configuration conf &＃61; new Configuration();
//GenericOptionsParser辅助工具类
   //String[] otherArgs &＃61; new GenericOptionsParser(conf, args).getRemainingArgs();
   String[] otherArgs &＃61; {"hdfs://centos:9000/kane/mini.txt","hdfs://centos:9000/kane/output"};
   if (otherArgs.length !&＃61; 2) {
     System.err.println("Usage: wordcount ");
     System.exit(2);
   }
   Job job &＃61; new Job(conf, "word count");
   job.setJarByClass(TestDefault.class);
   //中间的内容省略就採用默认的类操作,应该是原样输出
  
   FileInputFormat.addInputPath(job, new Path(otherArgs[0]));//输入參数&＃xff0c;相应hadoop jar 相应类执行时在后面加的第一个參数
   FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));//输出參数
   System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}

然后到处该类为jar包&＃xff0c;放到hadoop文件下&＃xff0c;执行

接下来自己创建须要mr执行的源文件&＃xff0c;并导入hdfs中

当我们执行hadoop命令执行时 可能出异常&＃xff0c;由于你编写代码的jdk可能和hadoop用到的JVM不匹配

默认的mr程序原样输出

測试wordcount

package com.kane.mr;


import java.io.IOException;
import java.util.StringTokenizer;


import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


public class MapperClass extends Mapper{
public Text keyText&＃61;new Text("key");
public IntWritable intValue&＃61;new IntWritable(1);
&＃64;Override
protected void map(Object key, Text value,
Context context)
throws IOException, InterruptedException {
//获取输入的值
String str&＃61;value.toString();
//用什么分隔键值&＃xff0c;默认空格或\t 或\n
StringTokenizer sTokenizer&＃61;new StringTokenizer(str);
//循环输出&＃xff0c;假如是My name is kane 则分四次输出四个单词
while (sTokenizer.hasMoreElements()) {
Object object &＃61; (Object) sTokenizer.nextElement();
//这里每一个单词能够看做一个key
keyText.set(str);
context.write(keyText, intValue);//匹配一个就加value比如&＃xff08;“My”&＃xff0c;1&＃xff09;
}
}

}

package com.kane.mr;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
//map传来的键值就是text 和 intwritable
public class ReducerClass extends Reducer{
public IntWritable intValue&＃61; new IntWritable(0);


&＃64;Override
protected void reduce(Text key, Iterable values,//假如name出现两次&＃xff0c;这里得到的values是 name [1,1]
Context context)
throws IOException, InterruptedException {
int sum&＃61;0;
while (values.iterator().hasNext()) {
sum&＃43;&＃61;values.iterator().next().get();

}
//这里值用intwritable输出是由于非常多情况下一个mapreduce的输出是下一个mapreduce的输入
intValue.set(sum);
context.write(key, intValue);
}

}

package com.kane.mr;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;


public class WordCounter {
public static void main(String[] args) throws Exception {
Configuration conf &＃61; new Configuration();
   String[] otherArgs &＃61; new GenericOptionsParser(conf, args).getRemainingArgs();
   if (otherArgs.length !&＃61; 2) {
     System.err.println("Usage: wordcount ");
     System.exit(2);
   }
   Job job &＃61; new Job(conf, "word count");
   job.setJarByClass(WordCounter.class);
   job.setMapperClass(MapperClass.class);
   //job.setCombinerClass(IntSumReducer.class);
   job.setReducerClass(ReducerClass.class);
   job.setOutputKeyClass(Text.class);
   job.setOutputValueClass(IntWritable.class);
   FileInputFormat.addInputPath(job, new Path(otherArgs[0]));//输入參数&＃xff0c;相应hadoop jar 相应类执行时在后面加的第一个參数
   FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));//输出參数
   System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}