@Override public Void answer(InvocationOnMock invocation) throws Throwable { conf.setStrings(MR_JAR_PROPERTY, "file:///tmp/foo1.jar", "file:///tmp/foo2.jar"); return null; }
private static void configureAvroShuffle(JobConf job) { job.setOutputKeyComparatorClass(AvroKeyComparator.class); job.setMapOutputKeyClass(AvroKey.class); job.setMapOutputValueClass(AvroValue.class); // add AvroSerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(AvroSerialization.class.getName())) { serializations.add(AvroSerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } }
jobConf.setStrings(MR_JAR_PROPERTY, new String[0]); for (BaseWork work : workGraph.keySet()) { work.configureJobConf(jobConf); jobConf.setStrings(MR_JAR_PROPERTY, finalTmpJars); return finalTmpJars;
jobConf.setStrings(MR_JAR_PROPERTY, new String[0]); for (BaseWork work : workGraph.keySet()) { work.configureJobConf(jobConf); jobConf.setStrings(MR_JAR_PROPERTY, finalTmpJars); return finalTmpJars;
ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
job.setMapOutputKeyClass(outputKeyClass); job.setMapperClass(mapper); job.setStrings("io.serializations", job.get("io.serializations"), MutationSerialization.class.getName(), ResultSerialization.class.getName()); FileInputFormat.addInputPaths(job, table);
job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(Put.class); job.setStrings("io.serializations", job.get("io.serializations"), MutationSerialization.class.getName(), ResultSerialization.class.getName()); if (partitioner == HRegionPartitioner.class) {
private static void setupTetherJob(JobConf job) throws IOException { job.setMapRunnerClass(TetherMapRunner.class); job.setPartitionerClass(TetherPartitioner.class); job.setReducerClass(TetherReducer.class); job.setInputFormat(TetherInputFormat.class); job.setOutputFormat(TetherOutputFormat.class); job.setOutputKeyClass(TetherData.class); job.setOutputKeyComparatorClass(TetherKeyComparator.class); job.setMapOutputValueClass(NullWritable.class); // set the map output key class to TetherData job.setMapOutputKeyClass(TetherData.class); // if protocol isn't set if (job.getStrings(TETHER_PROTOCOL)==null) { job.set(TETHER_PROTOCOL, "sasl"); } // add TetherKeySerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(TetherKeySerialization.class.getName())) { serializations.add(TetherKeySerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } // determine whether the executable should be added to the cache. if (job.getBoolean(TETHER_EXEC_CACHED,false)){ DistributedCache.addCacheFile(getExecutable(job), job); } }
conf.setBoolean(VoldemortBuildAndPushJob.SAVE_KEYS, saveKeys); conf.setBoolean(VoldemortBuildAndPushJob.REDUCER_OUTPUT_COMPRESS, true); conf.setStrings(VoldemortBuildAndPushJob.REDUCER_OUTPUT_COMPRESS_CODEC, KeyValueWriter.COMPRESSION_CODEC);
void setInputFieldNames(String... fieldNames) { job.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, fieldNames); }
void setOutputFieldNames(String... fieldNames) { job.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, fieldNames); }
void setOutputFieldNames(String... fieldNames) { job.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, fieldNames); }
void setInputFieldNames(String... fieldNames) { job.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, fieldNames); }
testDir.deleteOnExit(); JobConf conf = new JobConf(); conf.setStrings(MRConfig.LOCAL_DIR, "file://" + tmpPath); conf.setLong(MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES, 1024L); conf.setBoolean(MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED,
/** * Set the array of string values for the <code>name</code> property as as comma delimited values. * * @param name property name. * @param values The values */ @Override public void setStrings( String name, String... values ) { getJobConf().setStrings( name, values ); }
public void setReducers(Collection<String> reducers) { String reducerlist[] = getReducers(reducers); conf.setStrings("retriever.reducers", reducerlist); setNumReduceTasks( reducerlist.length ); }
private static void configureAvroShuffle(JobConf job) { job.setOutputKeyComparatorClass(AvroKeyComparator.class); job.setMapOutputKeyClass(AvroKey.class); job.setMapOutputValueClass(AvroValue.class); // add AvroSerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(AvroSerialization.class.getName())) { serializations.add(AvroSerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } }
public void setReducers(Collection<String> reducers) { String reducerlist[] = getReducers(reducers); conf.setStrings("retriever.reducers", reducerlist); setNumReduceTasks( reducerlist.length ); }
private static void configureAvroJob(JobConf job) { if (job.get("mapred.input.format.class") == null) job.setInputFormat(AvroInputFormat.class); if (job.get("mapred.output.format.class") == null) job.setOutputFormat(AvroOutputFormat.class); job.setOutputKeyClass(AvroWrapper.class); job.setOutputKeyComparatorClass(AvroKeyComparator.class); job.setMapOutputKeyClass(AvroKey.class); job.setMapOutputValueClass(AvroValue.class); job.setMapperClass(HadoopMapper.class); job.setReducerClass(HadoopReducer.class); // add AvroSerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(AvroSerialization.class.getName())) { serializations.add(AvroSerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } }
private static void setupTetherJob(JobConf job) throws IOException { job.setMapRunnerClass(TetherMapRunner.class); job.setPartitionerClass(TetherPartitioner.class); job.setReducerClass(TetherReducer.class); job.setInputFormat(TetherInputFormat.class); job.setOutputFormat(TetherOutputFormat.class); job.setOutputKeyClass(TetherData.class); job.setOutputKeyComparatorClass(TetherKeyComparator.class); job.setMapOutputValueClass(NullWritable.class); // add TetherKeySerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(TetherKeySerialization.class.getName())) { serializations.add(TetherKeySerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } DistributedCache.addCacheFile(getExecutable(job), job); }