/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass("mapred.map.output.compression.codec", codecClass, CompressionCodec.class); }
/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass(JobContext.MAP_OUTPUT_COMPRESS_CODEC, codecClass, CompressionCodec.class); }
/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass(JobContext.MAP_OUTPUT_COMPRESS_CODEC, codecClass, CompressionCodec.class); }
/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass(JobContext.MAP_OUTPUT_COMPRESS_CODEC, codecClass, CompressionCodec.class); }
/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass(JobContext.MAP_OUTPUT_COMPRESS_CODEC, codecClass, CompressionCodec.class); }
/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass("mapred.map.output.compression.codec", codecClass, CompressionCodec.class); }
/** * Set the given class as the {@link CompressionCodec} for the map outputs. * * @param codecClass the {@link CompressionCodec} class that will compress * the map outputs. */ public void setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) { setCompressMapOutput(true); setClass(JobContext.MAP_OUTPUT_COMPRESS_CODEC, codecClass, CompressionCodec.class); }
@SuppressWarnings("deprecation") private void initAvro() { this.setJarByClass(AvroJob.class); this.setSortComparatorClass(AvroKeyComparator.class); this.getConfiguration().setStrings( "io.serializations", new String[]{WritableSerialization.class.getName(), AvroReflectDXSerialization.class.getName()}); this.conf.setCompressMapOutput(true); }
conf.setCombinerClass(IdentityReducer.class); conf.setCompressMapOutput(compressMapOutputs); SequenceFileOutputFormat.setOutputCompressionType(conf, redCompression); try {
conf.setCombinerClass(IdentityReducer.class); conf.setCompressMapOutput(compressMapOutputs); SequenceFileOutputFormat.setOutputCompressionType(conf, redCompression); try {
private static void runSort(JobConf jobConf, Path sortInput, Path sortOutput) throws Exception { // Set up the job jobConf.setJobName("null-sorter"); jobConf.setMapperClass(SinkMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setNumReduceTasks(2); jobConf.setInputFormat(SequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); jobConf.setOutputKeyClass(BytesWritable.class); jobConf.setOutputValueClass(BytesWritable.class); FileInputFormat.setInputPaths(jobConf, sortInput); FileOutputFormat.setOutputPath(jobConf, sortOutput); // Compress the intermediate map-outputs! jobConf.setCompressMapOutput(true); // Run the job JobClient.runJob(jobConf); }
conf.setOutputValueClass(Text.class); conf.setCompressMapOutput(true);