@Override public void process(InputStream in, OutputStream out) throws IOException { // Use a FilterableOutputStream to change 'InputStreamWritable' to 'BytesWritable' - see comment // above for an explanation of why we want to do this. final ByteFilteringOutputStream bwos = new ByteFilteringOutputStream(out); // TODO: Adding this filter could be dangerous... A Sequence File's header contains 3 bytes: "SEQ", // followed by 1 byte that is the Sequence File version, followed by 2 "entries." These "entries" // contain the size of the Key/Value type and the Key/Value type. So, we will be writing the // value type as InputStreamWritable -- which we need to change to BytesWritable. This means that // we must also change the "size" that is written, but replacing this single byte could be // dangerous. However, we know exactly what will be written to the header, and we limit this at one // replacement, so we should be just fine. bwos.addFilter(toReplace, replaceWith, 1); bwos.addFilter((byte) InputStreamWritable.class.getCanonicalName().length(), (byte) BytesWritable.class.getCanonicalName().length(), 1); try (final FSDataOutputStream fsDataOutputStream = new FSDataOutputStream(bwos, new Statistics("")); final SequenceFile.Writer writer = SequenceFile.createWriter(configuration, SequenceFile.Writer.stream(fsDataOutputStream), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(InputStreamWritable.class), SequenceFile.Writer.compression(compressionType, compressionCodec))) { processInputStream(in, flowFile, writer); } finally { watch.stop(); } } });
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @param metadata The metadata of the file. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec), Writer.metadata(metadata)); }
4096, (short) 1, len, null); Writer w = SequenceFile.createWriter(new Configuration(), Writer.stream(out), Writer.keyClass(RandomDatum.class), Writer.valueClass(RandomDatum.class),
Writer writer = createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec),
Writer writer = createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec),
Writer writer = createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec),
Writer writer = createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec),
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @param metadata The metadata of the file. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec), Writer.metadata(metadata)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @param metadata The metadata of the file. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec), Writer.metadata(metadata)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @param metadata The metadata of the file. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec), Writer.metadata(metadata)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @param metadata The metadata of the file. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec), Writer.metadata(metadata)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec)); }
/** * Construct the preferred type of 'raw' SequenceFile Writer. * @param conf The configuration. * @param out The stream on top which the writer is to be constructed. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)} * instead. */ @Deprecated public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec) throws IOException { return createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec)); }
/** * {@inheritDoc} */ @Override protected void prepareWrite(WritableByteChannel channel) throws Exception { LOG.debug("Opening new writer"); Configuration configuration = new Configuration(false); configuration.setStrings("io.serializations", writeOperation.serializationNames); FSDataOutputStream outputStream = new FSDataOutputStream(new OutputStreamWrapper(channel), new Statistics("dataflow")); sequenceFile = SequenceFile.createWriter(configuration, SequenceFile.Writer.stream(outputStream), SequenceFile.Writer.keyClass(writeOperation.keyClass), SequenceFile.Writer.valueClass(writeOperation.valueClass), SequenceFile.Writer.compression(SequenceFile.CompressionType.BLOCK) ); }
, SequenceFile.Writer.keyClass(LongWritable.class) , SequenceFile.Writer.valueClass(BytesWritable.class) , SequenceFile.Writer.stream(outputStream) , SequenceFile.Writer.compression(SequenceFile.CompressionType.NONE) );
Writer writer = createWriter(conf, Writer.stream(out), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compressionType, codec),
@Override public SequenceFileWriter<K, V> create(FSDataOutputStream out) throws IOException { org.apache.hadoop.fs.FSDataOutputStream stream = new org.apache.hadoop.fs.FSDataOutputStream(out, null); CompressionCodec compressionCodec = getCompressionCodec(serializableHadoopConfig.get(), compressionCodecName); SequenceFile.Writer writer = SequenceFile.createWriter( serializableHadoopConfig.get(), SequenceFile.Writer.stream(stream), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(valueClass), SequenceFile.Writer.compression(compressionType, compressionCodec)); return new SequenceFileWriter<>(writer); }