@Override public RecordWriter<BytesWritable, BytesWritable> getRecordWriter( TaskAttemptContext context) throws IOException { final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClass(context), getSequenceFileOutputValueClass(context)); return new RecordWriter<BytesWritable, BytesWritable>() { private WritableValueBytes wvaluebytes = new WritableValueBytes(); public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException { wvaluebytes.reset(bvalue); out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes); wvaluebytes.reset(null); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
@Override public void checkOutputSpecs(JobContext job) throws IOException { super.checkOutputSpecs(job); if (getCompressOutput(job) && getOutputCompressionType(job) == CompressionType.RECORD ) { throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat " + "doesn't support Record Compression" ); } } }
public void testSequenceOutputClassDefaultsToMapRedOutputClass() throws IOException { Job job = Job.getInstance(); // Setting Random class to test getSequenceFileOutput{Key,Value}Class job.setOutputKeyClass(FloatWritable.class); job.setOutputValueClass(BooleanWritable.class); assertEquals("SequenceFileOutputKeyClass should default to ouputKeyClass", FloatWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputKeyClass(job)); assertEquals("SequenceFileOutputValueClass should default to " + "ouputValueClass", BooleanWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputValueClass(job)); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputKeyClass(job, IntWritable.class ); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputValueClass(job, DoubleWritable.class ); assertEquals("SequenceFileOutputKeyClass not updated", IntWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputKeyClass(job)); assertEquals("SequenceFileOutputValueClass not updated", DoubleWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputValueClass(job)); }
SequenceFileAsBinaryOutputFormat.setSequenceFileOutputKeyClass(job, IntWritable.class ); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputValueClass(job, DoubleWritable.class ); SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true); SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); MapReduceTestUtil.createDummyMapTaskAttemptContext(job.getConfiguration()); OutputFormat<BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat(); OutputCommitter committer = outputFormat.getOutputCommitter(context); committer.setupJob(job);
SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true); SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); try { new SequenceFileAsBinaryOutputFormat().checkOutputSpecs(job); } catch (Exception e) { fail("Block compression should be allowed for " SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.RECORD); try { new SequenceFileAsBinaryOutputFormat().checkOutputSpecs(job); fail("Record compression should not be allowed for " + "SequenceFileAsBinaryOutputFormat");
public void testSequenceOutputClassDefaultsToMapRedOutputClass() throws IOException { Job job = Job.getInstance(); // Setting Random class to test getSequenceFileOutput{Key,Value}Class job.setOutputKeyClass(FloatWritable.class); job.setOutputValueClass(BooleanWritable.class); assertEquals("SequenceFileOutputKeyClass should default to ouputKeyClass", FloatWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputKeyClass(job)); assertEquals("SequenceFileOutputValueClass should default to " + "ouputValueClass", BooleanWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputValueClass(job)); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputKeyClass(job, IntWritable.class ); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputValueClass(job, DoubleWritable.class ); assertEquals("SequenceFileOutputKeyClass not updated", IntWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputKeyClass(job)); assertEquals("SequenceFileOutputValueClass not updated", DoubleWritable.class, SequenceFileAsBinaryOutputFormat.getSequenceFileOutputValueClass(job)); }
SequenceFileAsBinaryOutputFormat.setSequenceFileOutputKeyClass(job, IntWritable.class ); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputValueClass(job, DoubleWritable.class ); SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true); SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); MapReduceTestUtil.createDummyMapTaskAttemptContext(job.getConfiguration()); OutputFormat<BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat(); OutputCommitter committer = outputFormat.getOutputCommitter(context); committer.setupJob(job);
SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true); SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); try { new SequenceFileAsBinaryOutputFormat().checkOutputSpecs(job); } catch (Exception e) { fail("Block compression should be allowed for " SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.RECORD); try { new SequenceFileAsBinaryOutputFormat().checkOutputSpecs(job); fail("Record compression should not be allowed for " + "SequenceFileAsBinaryOutputFormat");
@Override public RecordWriter<BytesWritable, BytesWritable> getRecordWriter( TaskAttemptContext context) throws IOException { final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClass(context), getSequenceFileOutputValueClass(context)); return new RecordWriter<BytesWritable, BytesWritable>() { private WritableValueBytes wvaluebytes = new WritableValueBytes(); public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException { wvaluebytes.reset(bvalue); out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes); wvaluebytes.reset(null); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
@Override public void checkOutputSpecs(JobContext job) throws IOException { super.checkOutputSpecs(job); if (getCompressOutput(job) && getOutputCompressionType(job) == CompressionType.RECORD ) { throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat " + "doesn't support Record Compression" ); } } }
@Override public RecordWriter<BytesWritable, BytesWritable> getRecordWriter( TaskAttemptContext context) throws IOException { final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClass(context), getSequenceFileOutputValueClass(context)); return new RecordWriter<BytesWritable, BytesWritable>() { private WritableValueBytes wvaluebytes = new WritableValueBytes(); public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException { wvaluebytes.reset(bvalue); out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes); wvaluebytes.reset(null); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
@Override public void checkOutputSpecs(JobContext job) throws IOException { super.checkOutputSpecs(job); if (getCompressOutput(job) && getOutputCompressionType(job) == CompressionType.RECORD ) { throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat " + "doesn't support Record Compression" ); } } }
@Override public RecordWriter<BytesWritable, BytesWritable> getRecordWriter( TaskAttemptContext context) throws IOException { final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClass(context), getSequenceFileOutputValueClass(context)); return new RecordWriter<BytesWritable, BytesWritable>() { private WritableValueBytes wvaluebytes = new WritableValueBytes(); public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException { wvaluebytes.reset(bvalue); out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes); wvaluebytes.reset(null); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
@Override public void checkOutputSpecs(JobContext job) throws IOException { super.checkOutputSpecs(job); if (getCompressOutput(job) && getOutputCompressionType(job) == CompressionType.RECORD ) { throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat " + "doesn't support Record Compression" ); } } }
@Override public RecordWriter<BytesWritable, BytesWritable> getRecordWriter( TaskAttemptContext context) throws IOException { final SequenceFile.Writer out = getSequenceWriter(context, getSequenceFileOutputKeyClass(context), getSequenceFileOutputValueClass(context)); return new RecordWriter<BytesWritable, BytesWritable>() { private WritableValueBytes wvaluebytes = new WritableValueBytes(); public void write(BytesWritable bkey, BytesWritable bvalue) throws IOException { wvaluebytes.reset(bvalue); out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes); wvaluebytes.reset(null); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
@Override public void checkOutputSpecs(JobContext job) throws IOException { super.checkOutputSpecs(job); if (getCompressOutput(job) && getOutputCompressionType(job) == CompressionType.RECORD ) { throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat " + "doesn't support Record Compression" ); } } }