StreamEventBuffer() { this.buffer = ByteBuffers.EMPTY_BUFFER; this.bufferInput = new ByteBufferInputStream(buffer); this.decoder = new BinaryDecoder(bufferInput); this.basePosition = -1L; }
private T decode(byte[] bytes) { if (bytes == null) { return null; } // decode T using schema ByteArrayInputStream bis = new ByteArrayInputStream(bytes); BinaryDecoder decoder = new BinaryDecoder(bis); try { return getReflectionDatumReader().read(decoder, this.schema); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to decode read object: " + e.getMessage(), e); } }
private T decode(byte[] bytes) { if (bytes == null) { return null; } // decode T using schema ByteArrayInputStream bis = new ByteArrayInputStream(bytes); BinaryDecoder decoder = new BinaryDecoder(bis); try { return getReflectionDatumReader().read(decoder, this.schema); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to decode read object: " + e.getMessage(), e); } }
@Override public AccessToken decode(byte[] data) throws IOException { ByteArrayInputStream bis = new ByteArrayInputStream(data); Decoder decoder = new BinaryDecoder(bis); DatumReader<AccessToken> reader = readerFactory.create(ACCESS_TOKEN_TYPE, AccessToken.Schemas.getCurrentSchema()); int readVersion = decoder.readInt(); Schema readSchema = AccessToken.Schemas.getSchemaVersion(readVersion); if (readSchema == null) { throw new IOException("Unknown schema version for AccessToken: " + readVersion); } return reader.read(decoder, readSchema); } }
@Override public AccessTokenIdentifier decode(byte[] data) throws IOException { ByteArrayInputStream bis = new ByteArrayInputStream(data); Decoder decoder = new BinaryDecoder(bis); DatumReader<AccessTokenIdentifier> reader = readerFactory.create(ACCESS_TOKEN_IDENTIFIER_TYPE, AccessTokenIdentifier.Schemas.getCurrentSchema()); int readVersion = decoder.readInt(); Schema readSchema = AccessTokenIdentifier.Schemas.getSchemaVersion(readVersion); if (readSchema == null) { throw new IOException("Unknown schema version for AccessTokenIdentifier: " + readVersion); } return reader.read(decoder, readSchema); }
@Override public KeyIdentifier decode(byte[] data) throws IOException { ByteArrayInputStream bis = new ByteArrayInputStream(data); Decoder decoder = new BinaryDecoder(bis); DatumReader<KeyIdentifier> reader = readerFactory.create(KEY_IDENTIFIER_TYPE, KeyIdentifier.Schemas.getCurrentSchema()); int readVersion = decoder.readInt(); Schema readSchema = KeyIdentifier.Schemas.getSchemaVersion(readVersion); if (readSchema == null) { throw new IOException("Unknown schema version for KeyIdentifier: " + readVersion); } return reader.read(decoder, readSchema); } }
@Test public void testMap() throws IOException, UnsupportedTypeException { TypeToken<Map<String, List<String>>> type = new TypeToken<Map<String, List<String>>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Map<String, List<String>>> writer = getWriter(type); ImmutableMap<String, List<String>> map = ImmutableMap.<String, List<String>>of("k1", Lists.newArrayList("v1"), "k2", Lists.newArrayList("v2", null)); writer.encode(map, new BinaryEncoder(os)); ReflectionDatumReader<Map<String, List<String>>> reader = new ReflectionDatumReader<>(getSchema(type), type); Assert.assertEquals(map, reader.read(new BinaryDecoder(is), getSchema(type))); }
@Test public void testURI() throws IOException, UnsupportedTypeException { TypeToken<List<URI>> type = new TypeToken<List<URI>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<List<URI>> writer = getWriter(type); List<URI> writeValue = ImmutableList.of(URI.create("http://www.abc.com")); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<URI>> reader = new ReflectionDatumReader<>(getSchema(type), type); Assert.assertEquals(writeValue, reader.read(new BinaryDecoder(is), getSchema(type))); }
@Test public void testRecordContainer() throws IOException, UnsupportedTypeException { TypeToken<List<Record>> type = new TypeToken<List<Record>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<List<Record>> writer = getWriter(type); List<Record> writeValue = ImmutableList.of( new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2)); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<Record>> reader = new ReflectionDatumReader<>(getSchema(type), type); List<Record> value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testUUID() throws UnsupportedTypeException, IOException { TypeToken<UUID> type = new TypeToken<UUID>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<UUID> writer = getWriter(type); UUID uuid = UUID.randomUUID(); writer.encode(uuid, new BinaryEncoder(os)); ReflectionDatumReader<UUID> reader = new ReflectionDatumReader<>(getSchema(type), type); UUID value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(uuid, value); }
@Test public void testList() throws IOException, UnsupportedTypeException { TypeToken<List<Long>> type = new TypeToken<List<Long>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); List<Long> writeValue = ImmutableList.of(1L, 10L, 100L, 1000L); DatumWriter<List<Long>> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<Long>> reader = new ReflectionDatumReader<>(getSchema(type), type); List<Long> value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testRecord() throws IOException, UnsupportedTypeException { TypeToken<Record> type = new TypeToken<Record>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Record> writer = getWriter(type); Record writeValue = new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<Record> reader = new ReflectionDatumReader<>(getSchema(type), type); Record value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testRecordArray() throws IOException, UnsupportedTypeException { TypeToken<Record[][]> type = new TypeToken<Record[][]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Record[][]> writer = getWriter(type); Record[][] writeValue = new Record[][] {{ new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2)}}; writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<Record[][]> reader = new ReflectionDatumReader<>(getSchema(type), type); Record[][] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testShort() throws UnsupportedTypeException, IOException { TypeToken<Short> type = new TypeToken<Short>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Short> writer = getWriter(type); writer.encode((short) 3000, new BinaryEncoder(os)); ReflectionDatumReader<Short> reader = new ReflectionDatumReader<>(getSchema(type), type); short value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals((short) 3000, value); }
@Test public void testString() throws UnsupportedTypeException, IOException { TypeToken<String> type = new TypeToken<String>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<String> writer = getWriter(type); writer.encode("Testing message", new BinaryEncoder(os)); ReflectionDatumReader<String> reader = new ReflectionDatumReader<>(getSchema(type), type); String value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals("Testing message", value); }
@Test public void testPrimitiveArray() throws IOException, UnsupportedTypeException { TypeToken<int[]> type = new TypeToken<int[]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); int[] writeValue = {1, 2, 3, 4, -5, -6, -7, -8}; DatumWriter<int[]> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<int[]> reader = new ReflectionDatumReader<>(getSchema(type), type); int[] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testReferenceArray() throws IOException, UnsupportedTypeException { TypeToken<String[]> type = new TypeToken<String[]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); String[] writeValue = new String[] {"1", "2", null, "3"}; DatumWriter<String[]> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<String[]> reader = new ReflectionDatumReader<>(getSchema(type), type); String[] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testInt() throws UnsupportedTypeException, IOException { TypeToken<Integer> type = new TypeToken<Integer>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Integer> writer = getWriter(type); writer.encode(12234234, new BinaryEncoder(os)); ReflectionDatumReader<Integer> reader = new ReflectionDatumReader<>(getSchema(type), type); int value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(12234234, value); }
@Test public void testDouble() throws UnsupportedTypeException, IOException { TypeToken<Double> type = new TypeToken<Double>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Double> writer = getWriter(type); writer.encode(3.14d, new BinaryEncoder(os)); ReflectionDatumReader<Double> reader = new ReflectionDatumReader<>(getSchema(type), type); double value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(3.14d, value, 0.000001d); }
@Test public void testReduceProjection() throws IOException, UnsupportedTypeException { PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); Schema sourceSchema = new ReflectionSchemaGenerator().generate(MoreFields.class); Schema targetSchema = new ReflectionSchemaGenerator().generate(LessFields.class); MoreFields moreFields = new MoreFields(10, 20.2, "30", ImmutableList.of("1", "2")); new ReflectionDatumWriter<MoreFields>(sourceSchema).encode(moreFields, new BinaryEncoder(output)); LessFields lessFields = new ReflectionDatumReader<>(targetSchema, TypeToken.of(LessFields.class)) .read(new BinaryDecoder(input), sourceSchema); Assert.assertEquals("30", lessFields.k); Assert.assertEquals(moreFields.inner.b, lessFields.inner.b); }