/** * Create a hive schema from the given type using reflection. * * @param type type of class to derive the schema from. * @return hive schema that can be used in a create statement. * @throws UnsupportedTypeException if there is an unsupported type. */ public String toHiveSchema(Type type) throws UnsupportedTypeException { // false is to disallow recursive schemas, which hive can't handle return toHiveSchema(schemaGenerator.generate(type, false)); }
/** * Create a hive schema from the given type using reflection. * * @param type type of class to derive the schema from. * @return hive schema that can be used in a create statement. * @throws UnsupportedTypeException if there is an unsupported type. */ public String toHiveSchema(Type type) throws UnsupportedTypeException { // false is to disallow recursive schemas, which hive can't handle return toHiveSchema(schemaGenerator.generate(type, false)); }
public IntegerStore(String name, KeyValueTable kvTable) throws UnsupportedTypeException { super(name, kvTable, new TypeRepresentation(Integer.class), new ReflectionSchemaGenerator().generate(Integer.class)); }
private <T> Schema getSchema(TypeToken<T> type) throws UnsupportedTypeException { return new ReflectionSchemaGenerator().generate(type.getType()); }
@Test public void testGenerateSchema() throws UnsupportedTypeException { Schema schema = (new ReflectionSchemaGenerator()).generate((new TypeToken<Child<Node>>() { }).getType()); Gson gson = new GsonBuilder() .registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) .create(); Assert.assertEquals(schema, gson.fromJson(gson.toJson(schema), Schema.class)); }
/** * Creates properties for {@link ObjectStore} dataset instance. * * @param type type of objects to be stored in dataset * @return {@link DatasetProperties} for the dataset * @throws UnsupportedTypeException */ public static DatasetProperties objectStoreProperties(Type type, DatasetProperties props) throws UnsupportedTypeException { Schema schema = new ReflectionSchemaGenerator().generate(type); TypeRepresentation typeRep = new TypeRepresentation(type); return DatasetProperties.builder() .add("schema", schema.toString()) .add("type", new Gson().toJson(typeRep)) .addAll(props.getProperties()) .build(); }
@Test public void testCompatible() throws UnsupportedTypeException { Schema s1 = new ReflectionSchemaGenerator().generate(Node.class); Schema s2 = new ReflectionSchemaGenerator().generate(Node3.class); Schema s3 = new ReflectionSchemaGenerator().generate(Node4.class); Assert.assertNotEquals(s1, s2); Assert.assertTrue(s1.isCompatible(s2)); Assert.assertFalse(s2.isCompatible(s1)); Assert.assertTrue(s2.isCompatible(s3)); }
@Test public void testReduceProjection() throws IOException, UnsupportedTypeException { PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); Schema sourceSchema = new ReflectionSchemaGenerator().generate(MoreFields.class); Schema targetSchema = new ReflectionSchemaGenerator().generate(LessFields.class); MoreFields moreFields = new MoreFields(10, 20.2, "30", ImmutableList.of("1", "2")); new ReflectionDatumWriter<MoreFields>(sourceSchema).encode(moreFields, new BinaryEncoder(output)); LessFields lessFields = new ReflectionDatumReader<>(targetSchema, TypeToken.of(LessFields.class)) .read(new BinaryDecoder(input), sourceSchema); Assert.assertEquals("30", lessFields.k); Assert.assertEquals(moreFields.inner.b, lessFields.inner.b); }
@Test public void testSchemaHash() throws UnsupportedTypeException { Schema s1 = new ReflectionSchemaGenerator().generate(Node.class); Schema s2 = new ReflectionSchemaGenerator().generate(Node2.class); Assert.assertEquals(s1.getSchemaHash(), s2.getSchemaHash()); Assert.assertEquals(s1, s2); Schema schema = (new ReflectionSchemaGenerator()).generate((new TypeToken<Child<Node>>() { }).getType()); Assert.assertNotEquals(s1.getSchemaHash(), schema.getSchemaHash()); }
@Test public void testParseJson() throws IOException, UnsupportedTypeException { Schema schema = new ReflectionSchemaGenerator().generate(Node.class); Assert.assertEquals(schema, Schema.parseJson(schema.toString())); }
@Test(expected = IOException.class) public void testCircularRef() throws UnsupportedTypeException, IOException { Schema schema = new ReflectionSchemaGenerator().generate(Node.class); Node head = new Node(); head.next = new Node(); head.next.next = head; ByteArrayOutputStream output = new ByteArrayOutputStream(); new ReflectionDatumWriter<Node>(schema).encode(head, new BinaryEncoder(output)); }
@Test public void testPrimitiveArray() throws UnsupportedTypeException { Schema schema = new ReflectionSchemaGenerator().generate(int[].class); Assert.assertEquals(Schema.arrayOf(Schema.of(Schema.Type.INT)), schema); }
@Test public void testCollection() throws UnsupportedTypeException, IOException { List<String> list = Lists.newArrayList("1", "2", "3"); Schema sourceSchema = new ReflectionSchemaGenerator().generate(new TypeToken<List<String>>() { }.getType()); Schema targetSchema = new ReflectionSchemaGenerator().generate(new TypeToken<Set<String>>() { }.getType()); PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output)); Set<String> set = new ReflectionDatumReader<>(targetSchema, new TypeToken<Set<String>>() { }) .read(new BinaryDecoder(input), sourceSchema); Assert.assertEquals(Sets.newHashSet("1", "2", "3"), set); targetSchema = new ReflectionSchemaGenerator().generate(String[].class); new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output)); String[] array = new ReflectionDatumReader<>(targetSchema, new TypeToken<String[]>() { }) .read(new BinaryDecoder(input), sourceSchema); Assert.assertArrayEquals(new String[]{"1", "2", "3"}, array); }
@Test public void testSameRecordDifferentLevels() throws UnsupportedTypeException, IOException { Schema actual = new ReflectionSchemaGenerator().generate(Node6.class); Assert.assertEquals(Node6.SCHEMA, actual); // check serialization and deserialization. Assert.assertEquals(Node6.SCHEMA, Schema.parseJson(actual.toString())); }
@Test public void testEnum() throws IOException, UnsupportedTypeException { PipedOutputStream output = new PipedOutputStream(); PipedInputStream input = new PipedInputStream(output); Schema schema = new ReflectionSchemaGenerator().generate(TestEnum.class); ReflectionDatumWriter<TestEnum> writer = new ReflectionDatumWriter<>(schema); BinaryEncoder encoder = new BinaryEncoder(output); writer.encode(TestEnum.VALUE1, encoder); writer.encode(TestEnum.VALUE3, encoder); writer.encode(TestEnum.VALUE2, encoder); BinaryDecoder decoder = new BinaryDecoder(input); Schema readSchema = Schema.parseJson(schema.toString()); ReflectionDatumReader<TestEnum> reader = new ReflectionDatumReader<>(readSchema, TypeToken.of(TestEnum.class)); Assert.assertEquals(TestEnum.VALUE1, reader.read(decoder, readSchema)); Assert.assertEquals(TestEnum.VALUE3, reader.read(decoder, readSchema)); Assert.assertEquals(TestEnum.VALUE2, reader.read(decoder, readSchema)); }
@Test public void testPutAndGet() throws Exception { dsFrameworkUtil.createInstance("table", users, DatasetProperties.builder().build()); try { final Table usersTable = dsFrameworkUtil.getInstance(users); final byte[] rowKey = Bytes.toBytes(123); final Schema schema = new ReflectionSchemaGenerator().generate(User.class); assertGetAndPut(usersTable, rowKey, SAMUEL, schema); } finally { dsFrameworkUtil.deleteInstance(users); } }
@Test public void testEmptyValue() throws UnsupportedTypeException, IOException { Schema schema = new ReflectionSchemaGenerator().generate(RecordWithString.class); TypeRepresentation typeRep = new TypeRepresentation(RecordWithString.class); DatumWriter<RecordWithString> datumWriter = new ReflectionDatumWriter<>(schema); @SuppressWarnings("unchecked") ReflectionDatumReader<RecordWithString> datumReader = new ReflectionDatumReader<>( schema, (TypeToken<RecordWithString>) TypeToken.of(typeRep.toType())); RecordWithString record = new RecordWithString(); record.setA(42); record.setTheString(""); ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); datumWriter.encode(record, encoder); byte[] bytes = bos.toByteArray(); ByteArrayInputStream bis = new ByteArrayInputStream(bytes); BinaryDecoder decoder = new BinaryDecoder(bis); RecordWithString rec = datumReader.read(decoder, schema); Assert.assertEquals(record.getA(), rec.getA()); Assert.assertEquals(record.getTheString(), rec.getTheString()); } }
@SuppressWarnings("ConstantConditions") @Test public void testGeneratorRecursion() throws Exception { // This test the schema generator be able to handle recursive schema between multiple classes Schema generatedSchema = new ReflectionSchemaGenerator(false).generate(NodeParent.class); // Validate the NodeChild.parent schema is the same as the schema of the Parent class // The schema generator always generate array value as nullable union Schema childSchema = generatedSchema.getField("children").getSchema().getComponentSchema().getNonNullable(); Assert.assertEquals(generatedSchema, childSchema.getField("parent").getSchema()); // Serialize the schema and then parse it back Schema deserializedSchema = Schema.parseJson(generatedSchema.toString()); Assert.assertEquals(generatedSchema, deserializedSchema); }
@Test public void testStructuredRecordRepresentation() throws Exception { dsFrameworkUtil.createInstance("table", users, DatasetProperties.builder().build()); try { final Table usersTable = dsFrameworkUtil.getInstance(users); final byte[] rowKey = Bytes.toBytes(123); final Schema schema = new ReflectionSchemaGenerator().generate(User.class); // TableDataset is not accessible here, but we know that's the underlying implementation... TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor((TransactionAware) usersTable); tx.execute(new TransactionExecutor.Subroutine() { @Override public void apply() throws Exception { Put put = new Put(rowKey); ReflectionPutWriter<User> putWriter = new ReflectionPutWriter<>(schema); putWriter.write(SAMUEL, put); usersTable.put(put); Row row = usersTable.get(rowKey); ReflectionRowRecordReader rowReader = new ReflectionRowRecordReader(schema, null); StructuredRecord actual = rowReader.read(row, schema); assertRecordEqualsUser(SAMUEL, actual); } }); } finally { dsFrameworkUtil.deleteInstance(users); } }
@Test public void testNullFields() throws Exception { dsFrameworkUtil.createInstance("table", users, DatasetProperties.builder().build()); try { final Table usersTable = dsFrameworkUtil.getInstance(users); final byte[] rowKey = Bytes.toBytes(123); final Schema schema = new ReflectionSchemaGenerator().generate(User.class); assertGetAndPut(usersTable, rowKey, SAMUEL, schema); assertGetAndPut(usersTable, rowKey, SAMUEL_NO_TS, schema); assertGetAndPut(usersTable, rowKey, SAMUEL_NO_ID, schema); assertGetAndPut(usersTable, rowKey, SAMUEL_NO_FIRST, schema); assertGetAndPut(usersTable, rowKey, SAMUEL_NO_SALARY, schema); assertGetAndPut(usersTable, rowKey, SAMUEL_NO_PURCHASE, schema); assertGetAndPut(usersTable, rowKey, SAMUEL_NO_BLOB, schema); } finally { dsFrameworkUtil.deleteInstance(users); } }