开发笔记:mapreduce计数器

// 自定义枚举变量Enum
Counter counter = context.getCounter(Enum enum)

// 自己命名groupName和counterName
Counter counter = context.getCounter(String groupName,String counterName)

counter.setValue(long value);// 设置初始值

counter.increment(long incr);// 增加计数

Configuration cOnf= new Configuration();
Job job = new Job(conf, "MyCounter");
job.waitForCompletion(true);
Counters counters=job.getCounters();
Counter counter=counters.findCounter(LOG_PROCESSOR_COUNTER.BAD_RECORDS_LONG);// 查找枚举计数器，假如Enum的变量为BAD_RECORDS_LONG
long value=counter.getValue();//获取计数值

Configuration cOnf= new Configuration();
Job job = new Job(conf, "MyCounter");
job.waitForCompletion(true);
Counters counters = job.getCounters();
Counter counter=counters.findCounter("ErrorCounter","toolong");// 假如groupName为ErrorCounter，counterName为toolong
long value = counter.getValue();// 获取计数值

Configuration cOnf= new Configuration();
Job job = new Job(conf, "MyCounter");
job.waitForCompletion(true);
Counters counters=job.getCounters();
// 查找作业运行启动的reduce个数的计数器，groupName和counterName可以从内置计数器表格查询（前面已经列举有）
Counter counter=counters.findCounter("org.apache.hadoop.mapreduce.JobCounter","TOTAL_LAUNCHED_REDUCES");// 假如groupName为org.apache.hadoop.mapreduce.JobCounter，counterName为TOTAL_LAUNCHED_REDUCES
long value=counter.getValue();// 获取计数值

Configuration cOnf= new Configuration();
Job job = new Job(conf, "MyCounter");
Counters counters = job.getCounters();
for (CounterGroup group : counters) {
for (Counter counter : group) {
System.out.println(counter.getDisplayName() + ": " + counter.getName() + ": "+ counter.getValue());
}
}

package com.buaa;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* @ProjectName CustomCounterDemo
* @PackageName com.buaa
* @ClassName MyCounter
* @Description 假如一个文件，规范的格式是3个字段，“ ”作为分隔符，其中有2条异常数据，一条数据是只有2个字段，一条数据是有4个字段
* @Author 刘吉超
* @Date 2016-05-23 20:10:14
*/
public class MyCounter {
// 键
private static String TAB_SEPARATOR = " ";
public static class MyCounterMap extends
Mapper {
// 定义枚举对象
public static enum LOG_PROCESSOR_COUNTER {
BAD_RECORDS_LONG, BAD_RECORDS_SHORT
};

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String arr_value[] = value.toString().split(TAB_SEPARATOR);
if (arr_value.length > 3) {
/* 自定义计数器 */
context.getCounter("ErrorCounter", "toolong").increment(1);
/* 枚举计数器 */
context.getCounter(LOG_PROCESSOR_COUNTER.BAD_RECORDS_LONG).increment(1);
} else if (arr_value.length <3) {
// 自定义计数器
context.getCounter("ErrorCounter", "tooshort").increment(1);
// 枚举计数器
context.getCounter(LOG_PROCESSOR_COUNTER.BAD_RECORDS_SHORT).increment(1);
}
}
}
@SuppressWarnings("deprecation")
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
String[] args0 = {
"hdfs://hadoop2:9000/buaa/counter/counter.txt",
"hdfs://hadoop2:9000/buaa/counter/out/"
};
// 读取配置文件
Configuration cOnf= new Configuration();

// 如果输出目录存在，则删除
Path mypath = new Path(args0[1]);
FileSystem hdfs = mypath.getFileSystem(conf);
if (hdfs.isDirectory(mypath)) {
hdfs.delete(mypath, true);
}
// 新建一个任务
Job job = new Job(conf, "MyCounter");
// 主类
job.setJarByClass(MyCounter.class);
// Mapper
job.setMapperClass(MyCounterMap.class);
// 输入目录
FileInputFormat.addInputPath(job, new Path(args0[0]));
// 输出目录
FileOutputFormat.setOutputPath(job, new Path(args0[1]));

// 提交任务，并退出
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}