public WriteAvroResultWithExternalSchema(final Schema avroSchema, final RecordSchema recordSchema, final SchemaAccessWriter schemaAccessWriter, final OutputStream out, final BlockingQueue<BinaryEncoder> recycleQueue, final ComponentLog logger) { super(out); this.recordSchema = recordSchema; this.schemaAccessWriter = schemaAccessWriter; this.avroSchema = avroSchema; this.buffered = new BufferedOutputStream(out); this.recycleQueue = recycleQueue; BinaryEncoder reusableEncoder = recycleQueue.poll(); if (reusableEncoder == null) { logger.debug("Was not able to obtain a BinaryEncoder from reuse pool. This is normal for the first X number of iterations (where X is equal to the max size of the pool), " + "but if this continues, it indicates that increasing the size of the pool will likely yield better performance for this Avro Writer."); } encoder = EncoderFactory.get().blockingBinaryEncoder(buffered, reusableEncoder); datumWriter = new GenericDatumWriter<>(avroSchema); }
@Test public void testBlockingBinaryEncoderInit() throws IOException { OutputStream out = new ByteArrayOutputStream(); BinaryEncoder reuse = null; reuse = factory.blockingBinaryEncoder(out, reuse); Assert.assertSame(reuse, factory.blockingBinaryEncoder(out, reuse)); // comparison }
@Override protected Encoder newEncoder(ByteArrayOutputStream out) throws IOException { return new EncoderFactory().configureBlockSize(254).blockingBinaryEncoder(out, null); }
public static void checkBlockingBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
public Tests(int bufferSize, int depth, String input) throws IOException { this.depth = depth; byte[] in = input.getBytes("UTF-8"); JsonFactory f = new JsonFactory(); JsonParser p = f.createJsonParser( new ByteArrayInputStream(input.getBytes("UTF-8"))); ByteArrayOutputStream os = new ByteArrayOutputStream(); EncoderFactory factory = new EncoderFactory() .configureBlockSize(bufferSize); Encoder cos = factory.blockingBinaryEncoder(os, null); serialize(cos, p, os); cos.flush(); byte[] bb = os.toByteArray(); // dump(bb); this.input = DecoderFactory.get().binaryDecoder(bb, null); this.parser = f.createJsonParser(new ByteArrayInputStream(in)); }
public TestBlockingIO2 (int bufferSize, int skipLevel, String calls) throws IOException { ByteArrayOutputStream os = new ByteArrayOutputStream(); EncoderFactory factory = new EncoderFactory() .configureBlockSize(bufferSize); Encoder encoder = factory.blockingBinaryEncoder(os, null); this.values = TestValidatingIO.randomValues(calls); TestValidatingIO.generate(encoder, calls, values); encoder.flush(); byte[] bb = os.toByteArray(); decoder = DecoderFactory.get().binaryDecoder(bb, null); this.calls = calls; }
public static byte[] make(Schema sc, String calls, Object[] values, Encoding encoding) throws IOException { EncoderFactory factory = EncoderFactory.get(); ByteArrayOutputStream ba = new ByteArrayOutputStream(); Encoder bvo = null; switch (encoding) { case BINARY: bvo = factory.binaryEncoder(ba, null); break; case BLOCKING_BINARY: bvo = factory.blockingBinaryEncoder(ba, null); break; case JSON: bvo = factory.jsonEncoder(sc, ba); break; } Encoder vo = factory.validatingEncoder(sc, bvo); generate(vo, calls, values); vo.flush(); return ba.toByteArray(); }
@Test public void testBlockingBinaryEncoder() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); BinaryEncoder e = factory.blockingBinaryEncoder(baos, null); generateData(e, true); byte[] result = baos.toByteArray(); Assert.assertEquals(legacydata.length, result.length); Assert.assertArrayEquals(legacydata, result); baos.reset(); generateComplexData(e); byte[] result2 = baos.toByteArray(); // blocking will cause different length, should be two bytes larger Assert.assertEquals(complexdata.length + 2, result2.length); // the first byte is the array start, with the count of items negative Assert.assertEquals(complexdata[0] >>> 1, result2[0]); } }
@Test public void testNew() throws IOException { ByteBuffer payload = ByteBuffer.allocateDirect(8 * 1024); for (int i = 0; i < 500; i++) { payload.putInt(1); } payload.flip(); ByteBufferRecord bbr = new ByteBufferRecord(); bbr.setPayload(payload); bbr.setTp(TypeEnum.b); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ReflectDatumWriter<ByteBufferRecord> writer = new ReflectDatumWriter<ByteBufferRecord>(ByteBufferRecord.class); BinaryEncoder avroEncoder = EncoderFactory.get().blockingBinaryEncoder(outputStream, null); writer.write(bbr, avroEncoder); avroEncoder.flush(); byte[] bytes = outputStream.toByteArray(); ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); ReflectDatumReader<ByteBufferRecord> datumReader = new ReflectDatumReader<ByteBufferRecord>(ByteBufferRecord.class); BinaryDecoder avroDecoder = DecoderFactory.get().binaryDecoder(inputStream, null); ByteBufferRecord deserialized = datumReader.read(null, avroDecoder); Assert.assertEquals(bbr, deserialized); }
private static void checkBinary(ReflectData reflectData, Schema schema, Object datum, boolean equals, boolean blocking) throws IOException { ReflectDatumWriter<Object> writer = new ReflectDatumWriter<>(schema); ByteArrayOutputStream out = new ByteArrayOutputStream(); if (!blocking) { writer.write(datum, EncoderFactory.get().directBinaryEncoder(out, null)); } else { writer.write(datum, new EncoderFactory().configureBlockSize(64) .blockingBinaryEncoder(out, null)); } writer.write(datum, EncoderFactory.get().directBinaryEncoder(out, null)); byte[] data = out.toByteArray(); ReflectDatumReader<Object> reader = new ReflectDatumReader<>(schema); Object decoded = reader.read(null, DecoderFactory.get().binaryDecoder(data, null)); assertEquals(0, reflectData.compare(datum, decoded, schema, equals)); }
@Override public void write(OutputStream out, Bundle row) throws IOException { GenericRecord outputRecord = new GenericData.Record(outputSchema); populateAvroRecord(outputRecord, row); encoder = EncoderFactory.get().blockingBinaryEncoder(out, encoder); datumWriter.write(outputRecord, encoder); encoder.flush(); }