org.apache.hadoop.mapreduce.lib.input.MultipleInputs.addInputPath()方法的使用及代码示例

本文整理了Java中org.apache.hadoop.mapreduce.lib.input.MultipleInputs.addInputPath()方法的一些代码示例，展示了MultipleInputs.addInputPath()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。MultipleInputs.addInputPath()方法的具体详情如下：
包路径：org.apache.hadoop.mapreduce.lib.input.MultipleInputs
类名称：MultipleInputs
方法名：addInputPath

MultipleInputs.addInputPath介绍

[英]Add a Path with a custom InputFormat to the list of inputs for the map-reduce job.
[中]将带有自定义InputFormat的路径添加到map reduce作业的输入列表中。

代码示例

代码示例来源：origin: apache/incubator-druid

private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass)
{
Configuration cOnf= job.getConfiguration();
StringBuilder inputFormats = new StringBuilder(
StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS))
);
String[] paths = Iterables.toArray(pathStrings, String.class);
for (int i = 0; i if (inputFormats.length() > 0) {
inputFormats.append(',');
}
inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName());
}
if (inputFormats.length() > 0) {
conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString());
}
// add last one separately for possible initialization in MultipleInputs
MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass);
}

代码示例来源：origin: apache/incubator-druid

MultipleInputs.addInputPath(job, new Path("/dummy/tobe/ignored"), DatasourceInputFormat.class);
return job;

代码示例来源：origin: mahmoudparsian/data-algorithms-book

MultipleInputs.addInputPath(job, transactions, TextInputFormat.class, LeftJoinTransactionMapper.class);
MultipleInputs.addInputPath(job, users, TextInputFormat.class, LeftJoinUserMapper.class);

代码示例来源：origin: apache/incubator-rya

public static void initJoinMRJob(Job job, String prospectsPath, String spoPath, Class> mapperClass,
String outPath, String auths) throws AccumuloSecurityException {
MultipleInputs.addInputPath(job, new Path(prospectsPath), SequenceFileInputFormat.class, mapperClass);
MultipleInputs.addInputPath(job, new Path(spoPath), SequenceFileInputFormat.class, mapperClass);
job.setMapOutputKeyClass(CompositeType.class);
job.setMapOutputValueClass(TripleCard.class);
SequenceFileOutputFormat.setOutputPath(job, new Path(outPath));
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setOutputKeyClass(TripleEntry.class);
job.setOutputValueClass(CardList.class);
}

代码示例来源：origin: apache/incubator-rya

/**
* Set up the MapReduce job to use Accumulo as an input.
* @param tableMapper Mapper class to use
*/
protected void configureAccumuloInput(Class> tableMapper)
throws AccumuloSecurityException {
MRReasoningUtils.configureAccumuloInput(job);
MultipleInputs.addInputPath(job, new Path("/tmp/input"),
AccumuloInputFormat.class, tableMapper);
}

代码示例来源：origin: apache/incubator-rya

/**
* Set up the MapReduce job to use an RDF file as an input.
* @param rdfMapper class to use
*/
protected void configureRdfInput(Path inputPath,
Class> rdfMapper) {
Configuration cOnf= job.getConfiguration();
String format = conf.get(MRUtils.FORMAT_PROP, RDFFormat.RDFXML.getName());
conf.set(MRUtils.FORMAT_PROP, format);
MultipleInputs.addInputPath(job, inputPath,
RdfFileInputFormat.class, rdfMapper);
}

代码示例来源：origin: com.github.jiayuhan-it/hadoop-mapreduce-client-core

/**
* Add a {@link Path} with a custom {@link InputFormat} and
* {@link Mapper} to the list of inputs for the map-reduce job.
*
* @param job The {@link Job}
* @param path {@link Path} to be added to the list of inputs for the job
* @param inputFormatClass {@link InputFormat} class to use for this path
* @param mapperClass {@link Mapper} class to use for this path
*/
@SuppressWarnings("unchecked")
public static void addInputPath(Job job, Path path,
Class inputFormatClass,
Class mapperClass) {
addInputPath(job, path, inputFormatClass);
Configuration cOnf= job.getConfiguration();
String mapperMapping = path.toString() + ";" + mapperClass.getName();
String mappers = conf.get(DIR_MAPPERS);
conf.set(DIR_MAPPERS, mappers == null ? mapperMapping
: mappers + "," + mapperMapping);
job.setMapperClass(DelegatingMapper.class);
}

代码示例来源：origin: org.apache.hadoop/hadoop-mapred

/**
* Add a {@link Path} with a custom {@link InputFormat} and
* {@link Mapper} to the list of inputs for the map-reduce job.
*
* @param job The {@link Job}
* @param path {@link Path} to be added to the list of inputs for the job
* @param inputFormatClass {@link InputFormat} class to use for this path
* @param mapperClass {@link Mapper} class to use for this path
*/
@SuppressWarnings("unchecked")
public static void addInputPath(Job job, Path path,
Class inputFormatClass,
Class mapperClass) {
addInputPath(job, path, inputFormatClass);
Configuration cOnf= job.getConfiguration();
String mapperMapping = path.toString() + ";" + mapperClass.getName();
String mappers = conf.get(DIR_MAPPERS);
conf.set(DIR_MAPPERS, mappers == null ? mapperMapping
: mappers + "," + mapperMapping);
job.setMapperClass(DelegatingMapper.class);
}

代码示例来源：origin: org.apache.hadoop/hadoop-mapred-test

@SuppressWarnings("unchecked")
public void testAddInputPathWithFormat() throws IOException {
final Job cOnf= Job.getInstance();
MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class);
MultipleInputs.addInputPath(conf, new Path("/bar"),
KeyValueTextInputFormat.class);
final Map inputs = MultipleInputs
.getInputFormatMap(conf);
assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
.getClass());
}

代码示例来源：origin: geftimov/hadoop-map-reduce-patterns

@Override
public int run(String[] args) throws Exception {
Configuration cOnf= new Configuration();
Job job = new Job(conf, "PostCommentHeirarchy");
job.setJarByClass(PostCommentHierarchy.class);
MultipleInputs.addInputPath(job, new Path(args[0]),
TextInputFormat.class, PostMapper.class);
MultipleInputs.addInputPath(job, new Path(args[1]),
TextInputFormat.class, CommentMapper.class);
job.setReducerClass(PostCommentHierarchyReducer.class);
job.setOutputFormatClass(TextOutputFormat.class);
TextOutputFormat.setOutputPath(job, new Path(args[2]));
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
return job.waitForCompletion(true) ? 0 : 2;
}

代码示例来源：origin: io.druid/druid-indexing-hadoop

private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass)
{
Configuration cOnf= job.getConfiguration();
StringBuilder inputFormats = new StringBuilder(Strings.nullToEmpty(conf.get(MultipleInputs.DIR_FORMATS)));
String[] paths = Iterables.toArray(pathStrings, String.class);
for (int i = 0; i if (inputFormats.length() > 0) {
inputFormats.append(',');
}
inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName());
}
if (inputFormats.length() > 0) {
conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString());
}
// add last one separately for possible initialization in MultipleInputs
MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass);
}

代码示例来源：origin: io.prestosql.hadoop/hadoop-apache

/**
* Add a {@link Path} with a custom {@link InputFormat} and
* {@link Mapper} to the list of inputs for the map-reduce job.
*
* @param job The {@link Job}
* @param path {@link Path} to be added to the list of inputs for the job
* @param inputFormatClass {@link InputFormat} class to use for this path
* @param mapperClass {@link Mapper} class to use for this path
*/
@SuppressWarnings("unchecked")
public static void addInputPath(Job job, Path path,
Class inputFormatClass,
Class mapperClass) {
addInputPath(job, path, inputFormatClass);
Configuration cOnf= job.getConfiguration();
String mapperMapping = path.toString() + ";" + mapperClass.getName();
String mappers = conf.get(DIR_MAPPERS);
conf.set(DIR_MAPPERS, mappers == null ? mapperMapping
: mappers + "," + mapperMapping);
job.setMapperClass(DelegatingMapper.class);
}

代码示例来源：origin: io.hops/hadoop-mapreduce-client-core

/**
* Add a {@link Path} with a custom {@link InputFormat} and
* {@link Mapper} to the list of inputs for the map-reduce job.
*
* @param job The {@link Job}
* @param path {@link Path} to be added to the list of inputs for the job
* @param inputFormatClass {@link InputFormat} class to use for this path
* @param mapperClass {@link Mapper} class to use for this path
*/
@SuppressWarnings("unchecked")
public static void addInputPath(Job job, Path path,
Class inputFormatClass,
Class mapperClass) {
addInputPath(job, path, inputFormatClass);
Configuration cOnf= job.getConfiguration();
String mapperMapping = path.toString() + ";" + mapperClass.getName();
String mappers = conf.get(DIR_MAPPERS);
conf.set(DIR_MAPPERS, mappers == null ? mapperMapping
: mappers + "," + mapperMapping);
job.setMapperClass(DelegatingMapper.class);
}

代码示例来源：origin: ch.cern.hadoop/hadoop-mapreduce-client-core

/**
* Add a {@link Path} with a custom {@link InputFormat} and
* {@link Mapper} to the list of inputs for the map-reduce job.
*
* @param job The {@link Job}
* @param path {@link Path} to be added to the list of inputs for the job
* @param inputFormatClass {@link InputFormat} class to use for this path
* @param mapperClass {@link Mapper} class to use for this path
*/
@SuppressWarnings("unchecked")
public static void addInputPath(Job job, Path path,
Class inputFormatClass,
Class mapperClass) {
addInputPath(job, path, inputFormatClass);
Configuration cOnf= job.getConfiguration();
String mapperMapping = path.toString() + ";" + mapperClass.getName();
String mappers = conf.get(DIR_MAPPERS);
conf.set(DIR_MAPPERS, mappers == null ? mapperMapping
: mappers + "," + mapperMapping);
job.setMapperClass(DelegatingMapper.class);
}

代码示例来源：origin: org.apache.druid/druid-indexing-hadoop

private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass)
{
Configuration cOnf= job.getConfiguration();
StringBuilder inputFormats = new StringBuilder(
StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS))
);
String[] paths = Iterables.toArray(pathStrings, String.class);
for (int i = 0; i if (inputFormats.length() > 0) {
inputFormats.append(',');
}
inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName());
}
if (inputFormats.length() > 0) {
conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString());
}
// add last one separately for possible initialization in MultipleInputs
MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass);
}

代码示例来源：origin: rathboma/hadoop-framework-examples

protected static void runFirstJob(Path transactions, Path users, Path output, Configuration conf)
throws Exception {
Job job = new Job(conf);
job.setJarByClass(RawMapreduce.class);
job.setJobName("Raw Mapreduce Step 1");
job.setPartitionerClass(SecondarySort.SSPartitioner.class);
job.setGroupingComparatorClass(SecondarySort.SSGroupComparator.class);
job.setSortComparatorClass(SecondarySort.SSSortComparator.class);
job.setReducerClass(JoinReducer.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
MultipleInputs.addInputPath(job, transactions, TextInputFormat.class, TransactionMapper.class);
MultipleInputs.addInputPath(job, users, TextInputFormat.class, UserMapper.class);
job.setMapOutputKeyClass(TextTuple.class);
job.setMapOutputValueClass(TextTuple.class);
FileOutputFormat.setOutputPath(job, output);

if (job.waitForCompletion(true)) return;
else throw new Exception("First Job Failed");
}

代码示例来源：origin: org.apache.hadoop/hadoop-mapred-test

@SuppressWarnings("unchecked")
public void testAddInputPathWithMapper() throws IOException {
final Job cOnf= Job.getInstance();
MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class,
MapClass.class);
MultipleInputs.addInputPath(conf, new Path("/bar"),
KeyValueTextInputFormat.class, KeyValueMapClass.class);
final Map inputs = MultipleInputs
.getInputFormatMap(conf);
final Map> maps = MultipleInputs
.getMapperTypeMap(conf);
assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
.getClass());
assertEquals(MapClass.class, maps.get(new Path("/foo")));
assertEquals(KeyValueMapClass.class, maps.get(new Path("/bar")));
}

代码示例来源：origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

@SuppressWarnings("unchecked")
public void testAddInputPathWithFormat() throws IOException {
final Job cOnf= Job.getInstance();
MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class);
MultipleInputs.addInputPath(conf, new Path("/bar"),
KeyValueTextInputFormat.class);
final Map inputs = MultipleInputs
.getInputFormatMap(conf);
assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
.getClass());
}

代码示例来源：origin: com.conversantmedia/mara-core

@Override
public void process(Annotation annotation, Job job, Object target)
throws ToolException {
for (Input input : ((MultiInput)annotation).value()) {
Path path = getInputAsPath(input.path());
if (input.mapper() == Mapper.class) {
MultipleInputs.addInputPath(job, path, input.format());
}
else {
MultipleInputs.addInputPath(job, path, input.format(), input.mapper());
// Need to call again here so the call is captured by our aspect which
// will replace it with the annotated delegating mapper class for resource
// injection if required.
job.setMapperClass(DelegatingMapper.class);
}
}
}

代码示例来源：origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

@SuppressWarnings("unchecked")
public void testAddInputPathWithMapper() throws IOException {
final Job cOnf= Job.getInstance();
MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class,
MapClass.class);
MultipleInputs.addInputPath(conf, new Path("/bar"),
KeyValueTextInputFormat.class, KeyValueMapClass.class);
final Map inputs = MultipleInputs
.getInputFormatMap(conf);
final Map> maps = MultipleInputs
.getMapperTypeMap(conf);
assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
.getClass());
assertEquals(MapClass.class, maps.get(new Path("/foo")));
assertEquals(KeyValueMapClass.class, maps.get(new Path("/bar")));
}