public ObjectStoreDataset(String name, KeyValueTable kvTable, TypeRepresentation typeRep, Schema schema, @Nullable ClassLoader classLoader) { super(name, kvTable); this.kvTable = kvTable; this.typeRep = typeRep; this.typeRep.setClassLoader(classLoader); this.schema = schema; this.datumWriter = new ReflectionDatumWriter<>(this.schema); }
private byte[] encode(T object) { // encode T using schema ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); try { this.datumWriter.encode(object, encoder); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to encode object to be written: " + e.getMessage(), e); } return bos.toByteArray(); }
@Override public void encode(T data, Encoder encoder) throws IOException { write(data, encoder); }
@Test(expected = IOException.class) public void testCircularRef() throws UnsupportedTypeException, IOException { Schema schema = new ReflectionSchemaGenerator().generate(Node.class); Node head = new Node(); head.next = new Node(); head.next.next = head; ByteArrayOutputStream output = new ByteArrayOutputStream(); new ReflectionDatumWriter<Node>(schema).encode(head, new BinaryEncoder(output)); }
@Test public void testEnum() throws IOException, UnsupportedTypeException { PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); Schema schema = new ReflectionSchemaGenerator().generate(TestEnum.class); ReflectionDatumWriter<TestEnum> writer = new ReflectionDatumWriter<>(schema); BinaryEncoder encoder = new BinaryEncoder(output); writer.encode(TestEnum.VALUE1, encoder); writer.encode(TestEnum.VALUE3, encoder); writer.encode(TestEnum.VALUE2, encoder); BinaryDecoder decoder = new BinaryDecoder(input); Schema readSchema = Schema.parseJson(schema.toString()); ReflectionDatumReader<TestEnum> reader = new ReflectionDatumReader<>(readSchema, TypeToken.of(TestEnum.class)); Assert.assertEquals(TestEnum.VALUE1, reader.read(decoder, readSchema)); Assert.assertEquals(TestEnum.VALUE3, reader.read(decoder, readSchema)); Assert.assertEquals(TestEnum.VALUE2, reader.read(decoder, readSchema)); }
@Test public void testReduceProjection() throws IOException, UnsupportedTypeException { PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); Schema sourceSchema = new ReflectionSchemaGenerator().generate(MoreFields.class); Schema targetSchema = new ReflectionSchemaGenerator().generate(LessFields.class); MoreFields moreFields = new MoreFields(10, 20.2, "30", ImmutableList.of("1", "2")); new ReflectionDatumWriter<MoreFields>(sourceSchema).encode(moreFields, new BinaryEncoder(output)); LessFields lessFields = new ReflectionDatumReader<>(targetSchema, TypeToken.of(LessFields.class)) .read(new BinaryDecoder(input), sourceSchema); Assert.assertEquals("30", lessFields.k); Assert.assertEquals(moreFields.inner.b, lessFields.inner.b); }
public ObjectStoreDataset(String name, KeyValueTable kvTable, TypeRepresentation typeRep, Schema schema, @Nullable ClassLoader classLoader) { super(name, kvTable); this.kvTable = kvTable; this.typeRep = typeRep; this.typeRep.setClassLoader(classLoader); this.schema = schema; this.datumWriter = new ReflectionDatumWriter<>(this.schema); }
@Override public void encode(T data, Encoder encoder) throws IOException { write(data, encoder); }
private byte[] encode(T object) { // encode T using schema ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); try { this.datumWriter.encode(object, encoder); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to encode object to be written: " + e.getMessage(), e); } return bos.toByteArray(); }
@Test public void testCollection() throws UnsupportedTypeException, IOException { List<String> list = Lists.newArrayList("1", "2", "3"); Schema sourceSchema = new ReflectionSchemaGenerator().generate(new TypeToken<List<String>>() { }.getType()); Schema targetSchema = new ReflectionSchemaGenerator().generate(new TypeToken<Set<String>>() { }.getType()); PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output)); Set<String> set = new ReflectionDatumReader<>(targetSchema, new TypeToken<Set<String>>() { }) .read(new BinaryDecoder(input), sourceSchema); Assert.assertEquals(Sets.newHashSet("1", "2", "3"), set); targetSchema = new ReflectionSchemaGenerator().generate(String[].class); new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output)); String[] array = new ReflectionDatumReader<>(targetSchema, new TypeToken<String[]>() { }) .read(new BinaryDecoder(input), sourceSchema); Assert.assertArrayEquals(new String[]{"1", "2", "3"}, array); }
@Test public void testEmptyValue() throws UnsupportedTypeException, IOException { Schema schema = new ReflectionSchemaGenerator().generate(RecordWithString.class); TypeRepresentation typeRep = new TypeRepresentation(RecordWithString.class); DatumWriter<RecordWithString> datumWriter = new ReflectionDatumWriter<>(schema); @SuppressWarnings("unchecked") ReflectionDatumReader<RecordWithString> datumReader = new ReflectionDatumReader<>( schema, (TypeToken<RecordWithString>) TypeToken.of(typeRep.toType())); RecordWithString record = new RecordWithString(); record.setA(42); record.setTheString(""); ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); datumWriter.encode(record, encoder); byte[] bytes = bos.toByteArray(); ByteArrayInputStream bis = new ByteArrayInputStream(bytes); BinaryDecoder decoder = new BinaryDecoder(bis); RecordWithString rec = datumReader.read(decoder, schema); Assert.assertEquals(record.getA(), rec.getA()); Assert.assertEquals(record.getTheString(), rec.getTheString()); } }
@Override protected void writeMap(Encoder encoder, Map<?, ?> map, Map.Entry<Schema, Schema> mapSchema) throws IOException { int size = map.size(); encoder.writeInt(size); Schema keySchema = mapSchema.getKey(); Schema valSchema = mapSchema.getValue(); for (Map.Entry<?, ?> entry : map.entrySet()) { write(encoder, entry.getKey(), keySchema); write(encoder, entry.getValue(), valSchema); } if (size > 0) { encoder.writeInt(0); } }
System.out.println("Time spent: " + TimeUnit.MILLISECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS)); ReflectionDatumWriter<Node> datumWriter = new ReflectionDatumWriter<>(getSchema(type)); startTime = System.nanoTime(); for (int i = 0; i < 100000; i++) { os.reset(); datumWriter.encode(writeValue, new BinaryEncoder(os)); System.out.println("Time spent: " + TimeUnit.MILLISECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS)); datumWriter = new ReflectionDatumWriter<>(getSchema(type)); startTime = System.nanoTime(); for (int i = 0; i < 100000; i++) { os.reset(); datumWriter.encode(writeValue, new BinaryEncoder(os));
@Override protected void writeMap(Encoder encoder, Map<?, ?> map, Map.Entry<Schema, Schema> mapSchema) throws IOException { int size = map.size(); encoder.writeInt(size); Schema keySchema = mapSchema.getKey(); Schema valSchema = mapSchema.getValue(); for (Map.Entry<?, ?> entry : map.entrySet()) { write(encoder, entry.getKey(), keySchema); write(encoder, entry.getValue(), valSchema); } if (size > 0) { encoder.writeInt(0); } }
@Test public void testTypeProject() throws IOException, UnsupportedTypeException { final Record1 r1 = new Record1(10, Maps.<Integer, Value>newHashMap(), new URL("http://www.yahoo.com")); r1.properties.put(1, new Value(1, "Name1")); r1.properties.put(2, new Value(2, "Name2")); r1.properties.put(3, null); PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); Schema sourceSchema = new ReflectionSchemaGenerator().generate(Record1.class); Schema targetSchema = new ReflectionSchemaGenerator().generate(Record2.class); new ReflectionDatumWriter<Record1>(sourceSchema).encode(r1, new BinaryEncoder(output)); Record2 r2 = new ReflectionDatumReader<>(targetSchema, TypeToken.of(Record2.class)) .read(new BinaryDecoder(input), sourceSchema); Assert.assertEquals(10L, r2.i.longValue()); Assert.assertTrue(Iterables.all(r2.properties.entrySet(), new Predicate<Map.Entry<String, Value>>() { @Override public boolean apply(Map.Entry<String, Value> input) { Value value = r1.properties.get(Integer.valueOf(input.getKey())); return (value == null && input.getValue() == null) || (value.equals(input.getValue())); } })); Assert.assertNull(r2.name); Assert.assertArrayEquals(new long[] {1L, 2L}, r2.numbers); Assert.assertEquals(URI.create("http://www.yahoo.com"), r2.url); Assert.assertEquals(r1.uuid, r2.uuid); }
@Override protected void writeArray(Encoder encoder, Object array, Schema componentSchema) throws IOException { int size = Array.getLength(array); encoder.writeInt(size); for (int i = 0; i < size; i++) { write(encoder, Array.get(array, i), componentSchema); } if (size > 0) { encoder.writeInt(0); } }
@Override protected void writeArray(Encoder encoder, Object array, Schema componentSchema) throws IOException { int size = Array.getLength(array); encoder.writeInt(size); for (int i = 0; i < size; i++) { write(encoder, Array.get(array, i), componentSchema); } if (size > 0) { encoder.writeInt(0); } }
@Override protected void writeArray(Encoder encoder, Collection<?> col, Schema componentSchema) throws IOException { int size = col.size(); encoder.writeInt(size); for (Object obj : col) { write(encoder, obj, componentSchema); } if (size > 0) { encoder.writeInt(0); } }
@Override protected void writeArray(Encoder encoder, Collection<?> col, Schema componentSchema) throws IOException { int size = col.size(); encoder.writeInt(size); for (Object obj : col) { write(encoder, obj, componentSchema); } if (size > 0) { encoder.writeInt(0); } }
@Override protected void writeUnion(Encoder encoder, Object val, Schema schema) throws IOException { // Assumption in schema generation that index 0 is the object type, index 1 is null. if (val == null) { encoder.writeInt(1); } else { seenRefs.remove(val); encoder.writeInt(0); write(encoder, val, schema.getUnionSchema(0)); } } }