@Override public void writeString(Utf8 utf8) throws IOException { this.writeBytes(utf8.getBytes(), 0, utf8.getByteLength()); }
public void serialize(TetherData datum) throws IOException { encoder.writeBytes(datum.buffer()); encoder.flush(); //Flush shouldn't be required. Might be a bug in AVRO. }
@Override public void writeString(Utf8 utf8) throws IOException { this.writeBytes(utf8.getBytes(), 0, utf8.getByteLength()); }
/** Open a new file for data matching a schema with an explicit sync. */ public DataFileWriter<D> create(Schema schema, OutputStream outs, byte[] sync) throws IOException { assertNotOpen(); this.schema = schema; setMetaInternal(DataFileConstants.SCHEMA, schema.toString()); if (sync == null ) { this.sync = generateSync(); } else if (sync.length == 16) { this.sync = sync; } else { throw new IOException("sync must be exactly 16 bytes"); } init(outs); vout.writeFixed(DataFileConstants.MAGIC); // write magic vout.writeMapStart(); // write metadata vout.setItemCount(meta.size()); for (Map.Entry<String,byte[]> entry : meta.entrySet()) { vout.startItem(); vout.writeString(entry.getKey()); vout.writeBytes(entry.getValue()); } vout.writeMapEnd(); vout.writeFixed(this.sync); // write initial sync vout.flush(); //vout may be buffered, flush before writing to out return this; }
/** Open a new file for data matching a schema with an explicit sync. */ public DataFileWriter<D> create(Schema schema, OutputStream outs, byte[] sync) throws IOException { assertNotOpen(); this.schema = schema; setMetaInternal(DataFileConstants.SCHEMA, schema.toString()); if (sync == null ) { this.sync = generateSync(); } else if (sync.length == 16) { this.sync = sync; } else { throw new IOException("sync must be exactly 16 bytes"); } init(outs); vout.writeFixed(DataFileConstants.MAGIC); // write magic vout.writeMapStart(); // write metadata vout.setItemCount(meta.size()); for (Map.Entry<String,byte[]> entry : meta.entrySet()) { vout.startItem(); vout.writeString(entry.getKey()); vout.writeBytes(entry.getValue()); } vout.writeMapEnd(); vout.writeFixed(this.sync); // write initial sync vout.flush(); //vout may be buffered, flush before writing to out return this; }
@Test public void testReuse() throws IOException { ByteBufferOutputStream bbo1 = new ByteBufferOutputStream(); ByteBufferOutputStream bbo2 = new ByteBufferOutputStream(); byte[] b1 = new byte[] { 1, 2 }; BinaryEncoder e1 = e_factory.binaryEncoder(bbo1, null); e1.writeBytes(b1); e1.flush(); BinaryEncoder e2 = e_factory.binaryEncoder(bbo2, null); e2.writeBytes(b1); e2.flush(); DirectBinaryDecoder d = new DirectBinaryDecoder( new ByteBufferInputStream(bbo1.getBufferList())); ByteBuffer bb1 = d.readBytes(null); Assert.assertEquals(b1.length, bb1.limit() - bb1.position()); d.configure(new ByteBufferInputStream(bbo2.getBufferList())); ByteBuffer bb2 = d.readBytes(null); Assert.assertEquals(b1.length, bb2.limit() - bb2.position()); }
@Override public void writeBytes(byte[] bytes, int start, int len) throws IOException { wrappedEncoder.writeBytes(bytes, start, len); }
@Override public void writeBytes(ByteBuffer bytes) throws IOException { wrappedEncoder.writeBytes(bytes); }
@Override public void writeBytes(ByteBuffer bytes) throws IOException { wrappedEncoder.writeBytes(bytes); }
@Override public void writeBytes(byte[] bytes, int start, int len) throws IOException { wrappedEncoder.writeBytes(bytes, start, len); }
@Override public void writeBytes(byte[] bytes, int start, int len) throws IOException { wrappedEncoder.writeBytes(bytes, start, len); }
@Override public void writeBytes(ByteBuffer bytes) throws IOException { wrappedEncoder.writeBytes(bytes); }
public void serialize(TetherData datum) throws IOException { encoder.writeBytes(datum.buffer()); }
@Override public void writeString(Utf8 utf8) throws IOException { this.writeBytes(utf8.getBytes(), 0, utf8.getByteLength()); }
public void serialize(TetherData datum) throws IOException { encoder.writeBytes(datum.buffer()); encoder.flush(); //Flush shouldn't be required. Might be a bug in AVRO. }
@Override public void writeString(Utf8 utf8) throws IOException { this.writeBytes(utf8.getBytes(), 0, utf8.getByteLength()); }
public byte[] toBytes(Schema toSchema, Object o) { if (toSchema != null && toSchema.getType() == Type.UNION) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); org.apache.avro.io.BinaryEncoder avroEncoder = EncoderFactory.get().binaryEncoder(baos, null); int unionIndex = 0; try { if (o == null) { unionIndex = firstNullSchemaTypeIndex(toSchema); avroEncoder.writeIndex(unionIndex); avroEncoder.writeNull(); } else { unionIndex = firstNotNullSchemaTypeIndex(toSchema); avroEncoder.writeIndex(unionIndex); avroEncoder.writeBytes(toBytes(o)); } avroEncoder.flush(); return baos.toByteArray(); } catch (IOException e) { LOG.error(e.getMessage()); return toBytes(o); } } else { return toBytes(o); } }
/** Open a new file for data matching a schema with an explicit sync. */ public DataFileWriter<D> create(Schema schema, OutputStream outs, byte[] sync) throws IOException { assertNotOpen(); this.schema = schema; setMetaInternal(DataFileConstants.SCHEMA, schema.toString()); if (sync == null ) { this.sync = generateSync(); } else if (sync.length == 16) { this.sync = sync; } else { throw new IOException("sync must be exactly 16 bytes"); } init(outs); vout.writeFixed(DataFileConstants.MAGIC); // write magic vout.writeMapStart(); // write metadata vout.setItemCount(meta.size()); for (Map.Entry<String,byte[]> entry : meta.entrySet()) { vout.startItem(); vout.writeString(entry.getKey()); vout.writeBytes(entry.getValue()); } vout.writeMapEnd(); vout.writeFixed(this.sync); // write initial sync vout.flush(); //vout may be buffered, flush before writing to out return this; }
/** Open a new file for data matching a schema. */ public DataFileWriter<D> create(Schema schema, OutputStream outs) throws IOException { assertNotOpen(); this.schema = schema; setMetaInternal(DataFileConstants.SCHEMA, schema.toString()); this.sync = generateSync(); init(outs); vout.writeFixed(DataFileConstants.MAGIC); // write magic vout.writeMapStart(); // write metadata vout.setItemCount(meta.size()); for (Map.Entry<String,byte[]> entry : meta.entrySet()) { vout.startItem(); vout.writeString(entry.getKey()); vout.writeBytes(entry.getValue()); } vout.writeMapEnd(); vout.writeFixed(sync); // write initial sync vout.flush(); //vout may be buffered, flush before writing to out return this; }
/** Open a new file for data matching a schema. */ public DataFileWriter<D> create(Schema schema, OutputStream outs) throws IOException { assertNotOpen(); this.schema = schema; setMetaInternal(DataFileConstants.SCHEMA, schema.toString()); this.sync = generateSync(); init(outs); out.write(DataFileConstants.MAGIC); // write magic vout.writeMapStart(); // write metadata vout.setItemCount(meta.size()); for (Map.Entry<String,byte[]> entry : meta.entrySet()) { vout.startItem(); vout.writeString(entry.getKey()); vout.writeBytes(entry.getValue()); } vout.writeMapEnd(); vout.flush(); //vout may be buffered, flush before writing to out out.write(sync); // write initial sync return this; }