public static void useSortableBytesAsMapOutputKey(Job job, Class<? extends Partitioner> partitionerClass) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(partitionerClass); }
/** * Sets the map output key schema. * * @param job The job to configure. * @param schema The map output key schema. */ public static void setMapOutputKeySchema(Job job, Schema schema) { job.setMapOutputKeyClass(AvroKey.class); job.setGroupingComparatorClass(AvroKeyComparator.class); job.setSortComparatorClass(AvroKeyComparator.class); AvroSerialization.setKeyWriterSchema(job.getConfiguration(), schema); AvroSerialization.setKeyReaderSchema(job.getConfiguration(), schema); AvroSerialization.addToConfiguration(job.getConfiguration()); }
job.setMapOutputKeyClass(RowKeyWritable.class); job.setMapOutputValueClass(KeyValue.class); job.setSortComparatorClass(RowKeyWritable.RowKeyComparator.class);
FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT)); job.setSortComparatorClass(JavaSerializationComparator.class);
job.setSortComparatorClass(CompositeKeyComparator.class);
public int run(String[] args) throws Exception { Job job = new Job(getConf(), "count-dns-bases"); job.setJarByClass(FastaCountBaseDriver.class); job.setMapperClass(FastaCountBaseMapper.class); job.setReducerClass(FastaCountBaseReducer.class); job.setNumReduceTasks(5); // worked //job.setCombinerClass(FastaCountBaseCombiner.class); job.setInputFormatClass(FastaInputFormat.class); job.setPartitionerClass(BasePartitioner.class); // worked job.setSortComparatorClass(BaseComparator.class); // worked job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); boolean status = job.waitForCompletion(true); theLogger.info("run(): status="+status); return status ? 0 : 1; }
job.setSortComparatorClass(CompositeKeyComparator.class);
job.setSortComparatorClass(PairOfStrings.Comparator.class);
job.setJarByClass(HadoopGraph.class); if (mapSort.isPresent()) job.setSortComparatorClass(ObjectWritableComparator.ObjectWritableMapComparator.class); job.setMapperClass(HadoopMap.class); if (mapReduce.doStage(MapReduce.Stage.REDUCE)) { reduceSortJob.setSortComparatorClass(ObjectWritableComparator.ObjectWritableReduceComparator.class); reduceSortJob.setMapperClass(Mapper.class); reduceSortJob.setReducerClass(Reducer.class);
@Override public void setup(Job job) { job.setSortComparatorClass(LongWritable.Comparator.class); job.setNumReduceTasks(1); } @Override
@Override public void setup(Job job) { job.getConfiguration().setInt(WORDCOUNT_TOPN, topNWords); job.setSortComparatorClass(LongWritable.DecreasingComparator.class); job.setNumReduceTasks(1); }
@Override public void setup(Job job) { job.getConfiguration().setInt(WORDCOUNT_TOPN, topNWords); job.setSortComparatorClass(LongWritable.DecreasingComparator.class); job.setNumReduceTasks(1); }
@Override public void setup(Job job) { job.getConfiguration().setFloat(MAXP_KEY, (float) this.maxp); job.setSortComparatorClass(DoubleWritable.Comparator.class); } }
/** * Wraps the key comparator defined in the job with this {@link KeyComparatorWrapper} if it is defined. * @param job The MapReduce job */ static void wrap(Job job) { if (WrapperUtil.setIfDefined(job, MRJobConfig.KEY_COMPARATOR, ATTR_CLASS)) { job.setSortComparatorClass(KeyComparatorWrapper.class); } }
/** * Wraps the key comparator defined in the job with this {@link KeyComparatorWrapper} if it is defined. * @param job The MapReduce job */ static void wrap(Job job) { if (WrapperUtil.setIfDefined(job, MRJobConfig.KEY_COMPARATOR, ATTR_CLASS)) { job.setSortComparatorClass(KeyComparatorWrapper.class); } }
public static void useSortableBytesAsMapOutputKey(Job job) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(SortableBytesPartitioner.class); }
public static void useSortableBytesAsMapOutputKey(Job job, Class<? extends Partitioner> partitionerClass) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(partitionerClass); }
public void configure(Job job) { if (partitionerClass != null) { job.setPartitionerClass(partitionerClass); } if (groupingComparatorClass != null) { job.setGroupingComparatorClass(groupingComparatorClass); } if (sortComparatorClass != null) { job.setSortComparatorClass(sortComparatorClass); } if (numReducers > 0) { job.setNumReduceTasks(numReducers); } }
public static void useSortableBytesAsMapOutputKey(Job job) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(SortableBytesPartitioner.class); }
@Override public void process(Annotation annotation, Job job, Object target) { Sorter sorter = (Sorter)annotation; if (sorter != null && sorter.value() != null && sorter.value() != NULLCOMPARATOR.class) { job.setSortComparatorClass(sorter.value()); } } }