@Test public void testRecursiveRecord() { Schema schema = SchemaBuilder.record("LongList").fields() .name("value").type().longType().noDefault() .name("next").type().optional().type("LongList") .endRecord(); Assert.assertEquals("LongList", schema.getName()); List<Schema.Field> fields = schema.getFields(); Assert.assertEquals(2, fields.size()); Assert.assertEquals( new Schema.Field("value", Schema.create(Schema.Type.LONG), null, null), fields.get(0)); Assert.assertEquals( Schema.Type.UNION, fields.get(1).schema().getType()); Assert.assertEquals( Schema.Type.NULL, fields.get(1).schema().getTypes().get(0).getType()); Schema recordSchema = fields.get(1).schema().getTypes().get(1); Assert.assertEquals(Schema.Type.RECORD, recordSchema.getType()); Assert.assertEquals("LongList", recordSchema.getName()); Assert.assertEquals(NullNode.getInstance(), fields.get(1).defaultValue()); }
@Test public void testSwitchNamespace() { String originalNamespace = "originalNamespace"; String originalName = "originalName"; String newNamespace = "newNamespace"; Schema schema = SchemaBuilder.builder(originalNamespace).record(originalName).fields(). requiredDouble("double").optionalFloat("float").endRecord(); Map<String, String> map = Maps.newHashMap(); map.put(originalNamespace, newNamespace); Schema newSchema = AvroUtils.switchNamespace(schema, map); Assert.assertEquals(newSchema.getNamespace(), newNamespace); Assert.assertEquals(newSchema.getName(), originalName); for(Schema.Field field : newSchema.getFields()) { Assert.assertEquals(field, schema.getField(field.name())); } }
@Test public void testArray() { Schema longSchema = Schema.create(Schema.Type.LONG); Schema expected = Schema.createArray(longSchema); Schema schema1 = SchemaBuilder.array().items().longType(); Assert.assertEquals(expected, schema1); Schema schema2 = SchemaBuilder.array().items(longSchema); Assert.assertEquals(expected, schema2); Schema schema3 = SchemaBuilder.array().prop("p", "v") .items().type("long"); expected.addProp("p", "v"); Assert.assertEquals(expected, schema3); }
@Test public void testUnionAndNullable() { List<Schema> types = new ArrayList<>(); types.add(Schema.create(Schema.Type.LONG)); types.add(Schema.create(Schema.Type.NULL)); Schema expected = Schema.createUnion(types); Schema schema = SchemaBuilder.unionOf() .longType().and() .nullType().endUnion(); Assert.assertEquals(expected, schema); schema = SchemaBuilder.nullable().longType(); Assert.assertEquals(expected, schema); }
@Test public void testUnionWithNull() { Schema nestedRecord = SchemaBuilder.record("nested").fields().requiredDouble("double") .requiredString("string").endRecord(); Schema union = SchemaBuilder.unionOf().nullType().and().type(nestedRecord).endUnion(); Schema schema = SchemaBuilder.record("record").fields().name("union").type(union).noDefault().endRecord(); Schema doubleSchema = AvroUtils.getFieldSchema(schema, "union.double").get(); Assert.assertEquals(doubleSchema.getType(), Schema.Type.DOUBLE); GenericRecord nested = new GenericData.Record(nestedRecord); nested.put("double", 10); nested.put("string", "testString"); GenericRecord record = new GenericData.Record(schema); record.put("union", nested); String stringValue = AvroUtils.getFieldValue(record, "union.string").get().toString(); Assert.assertEquals(stringValue, "testString"); } }
public static File generateEnumTypes(String filename, int nrows, String[][] categories) throws IOException { assert categories.length == 2 : "Needs only 2 columns"; File parentDir = Files.createTempDir(); File f = new File(parentDir, filename); DatumWriter<GenericRecord> w = new GenericDatumWriter<GenericRecord>(); DataFileWriter<GenericRecord> dw = new DataFileWriter<GenericRecord>(w); Schema enumSchema1 = SchemaBuilder.enumeration("CEnum1").symbols(categories[0]); Schema enumSchema2 = SchemaBuilder.enumeration("CEnum2").symbols(categories[1]); Schema schema = SchemaBuilder.builder() .record("test_enum_types").fields() .name("CEnum").type(enumSchema1).noDefault() .name("CUEnum").type().optional().type(enumSchema2) .endRecord(); System.out.println(schema); int numOfCategories1 = categories[0].length; int numOfCategories2 = categories[1].length; try { dw.create(schema, f); for (int i = 0; i < nrows; i++) { GenericRecord gr = new GenericData.Record(schema); gr.put("CEnum", new GenericData.EnumSymbol(enumSchema1, categories[0][i % numOfCategories1])); gr.put("CUEnum", i % (numOfCategories2+1) == 0 ? null : new GenericData.EnumSymbol(enumSchema2, categories[1][i % numOfCategories2])); dw.append(gr); } return f; } finally { dw.close();; } }
@Test public void testRecord() { Schema schema = SchemaBuilder .record("myrecord").namespace("org.example").aliases("oldrecord").fields() .name("f0").aliases("f0alias").type().stringType().noDefault() .name("f1").doc("This is f1").type().longType().noDefault() .name("f2").type().nullable().booleanType().booleanDefault(true) .endRecord(); Assert.assertEquals("myrecord", schema.getName()); Assert.assertEquals("org.example", schema.getNamespace()); Assert.assertEquals("org.example.oldrecord", schema.getAliases().iterator().next()); Assert.assertFalse(schema.isError()); List<Schema.Field> fields = schema.getFields(); Assert.assertEquals(3, fields.size()); Assert.assertEquals( new Schema.Field("f0", Schema.create(Schema.Type.STRING), null, null), fields.get(0)); Assert.assertTrue(fields.get(0).aliases().contains("f0alias")); Assert.assertEquals( new Schema.Field("f1", Schema.create(Schema.Type.LONG), "This is f1", null), fields.get(1)); List<Schema> types = new ArrayList<>(); types.add(Schema.create(Schema.Type.BOOLEAN)); types.add(Schema.create(Schema.Type.NULL)); Schema optional = Schema.createUnion(types); Assert.assertEquals(new Schema.Field("f2", optional, null, true), fields.get(2)); }
@Test public void testReflectedSchema() { Schema expected = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); expected.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( expected.getField("uuids").schema().getElementType()); Schema actual = REFLECT.getSchema(RecordWithUUIDList.class); Assert.assertEquals("Should use the UUID logical type", expected, actual); }
@Test public void testSwitchName() { String originalName = "originalName"; String newName = "newName"; Schema schema = SchemaBuilder.record(originalName).fields(). requiredDouble("double").optionalFloat("float").endRecord(); Schema newSchema = AvroUtils.switchName(schema, newName); Assert.assertEquals(newSchema.getName(), newName); for(Schema.Field field : newSchema.getFields()) { Assert.assertEquals(field, schema.getField(field.name())); } Assert.assertEquals(newName, AvroUtils.switchName(schema, newName).getName()); Assert.assertEquals(schema, AvroUtils.switchName(AvroUtils.switchName(schema, newName), schema.getName())); }
@SuppressWarnings("fallthrough") private Schema getGranularityBasedSchema() { FieldAssembler<Schema> assembler = SchemaBuilder.record("GenericRecordTimePartition").namespace("gobblin.writer.partitioner").fields(); // Construct the fields in reverse order if (!Strings.isNullOrEmpty(this.writerPartitionSuffix)) { assembler = assembler.name(SUFFIX).type(Schema.create(Schema.Type.STRING)).noDefault(); } assembler = assembler.name(this.granularity.toString()).type(Schema.create(Schema.Type.STRING)).noDefault(); if (!Strings.isNullOrEmpty(this.writerPartitionPrefix)) { assembler = assembler.name(PREFIX).type(Schema.create(Schema.Type.STRING)).noDefault(); } Schema schema = assembler.endRecord(); Collections.reverse(schema.getFields()); return schema; }
@Test public void testAvroAliasOnField() { Schema expectedSchema = SchemaBuilder.record(ClassWithAliasOnField.class.getSimpleName()) .namespace("org.apache.avro.reflect.TestReflect").fields().name("primitiveField").aliases("aliasName") .type(Schema.create(org.apache.avro.Schema.Type.INT)).noDefault().endRecord(); check(ClassWithAliasOnField.class, expectedSchema.toString()); }
@Test(expected = DataFileWriter.AppendWriteException.class) public void testWriteUUIDMissingLogicalType() throws IOException { Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithUUID r1 = new RecordWithUUID(); r1.uuid = u1; RecordWithUUID r2 = new RecordWithUUID(); r2.uuid = u2; // write without using REFLECT, which has the logical type File test = write(uuidSchema, r1, r2); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); // this fails with an AppendWriteException wrapping ClassCastException // because the UUID isn't converted to a CharSequence expected internally read(ReflectData.get().createDatumReader(uuidStringSchema), test); }
@Test public void testReadUUIDArray() throws IOException { Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); LogicalTypes.uuid().addToSchema( uuidArraySchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidArraySchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDArray expected = new RecordWithUUIDArray(); expected.uuids = new UUID[] {u1, u2}; File test = write(uuidArraySchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidArraySchema), test).get(0)); }
@Test public void testNamespaces() { Schema s1 = SchemaBuilder.record("myrecord") .namespace("org.example") .fields() .name("myint").type().intType().noDefault() .endRecord(); Schema s2 = SchemaBuilder.record("org.example.myrecord") .fields() .name("myint").type().intType().noDefault() .endRecord(); Schema s3 = SchemaBuilder.record("org.example.myrecord") .namespace("org.example2") .fields() .name("myint").type().intType().noDefault() .endRecord(); Schema s4 = SchemaBuilder.builder("org.example").record("myrecord") .fields() .name("myint").type().intType().noDefault() .endRecord(); Assert.assertEquals("myrecord", s1.getName()); Assert.assertEquals("myrecord", s2.getName()); Assert.assertEquals("myrecord", s3.getName()); Assert.assertEquals("myrecord", s4.getName());
private Schema getDateTimeFormatBasedSchema() { FieldAssembler<Schema> assembler = SchemaBuilder.record("GenericRecordTimePartition").namespace("gobblin.writer.partitioner").fields(); if (!Strings.isNullOrEmpty(this.writerPartitionPrefix)) { assembler = assembler.name(PREFIX).type(Schema.create(Schema.Type.STRING)).noDefault(); } assembler = assembler.name(PARTITIONED_PATH).type(Schema.create(Schema.Type.STRING)).noDefault(); if (!Strings.isNullOrEmpty(this.writerPartitionSuffix)) { assembler = assembler.name(SUFFIX).type(Schema.create(Schema.Type.STRING)).noDefault(); } return assembler.endRecord(); }
array = new GenericData.Array<>(subject.getSchema().getField(shortHandPredicateURI).schema().getTypes().get(0), new LinkedList<GenericRecord>()); array.add((GenericRecord)object); array.add((GenericRecord)currentObj); strings = new GenericData.Array<>(SchemaBuilder.array().items().stringType(), new LinkedList<String>()); strings.add((String)object); strings.add((String)currentObj); strings = new GenericData.Array<>(SchemaBuilder.array().items().stringType(), new LinkedList<String>()); strings.add(objectToAdd);
@Test public void testSplitSchemaBuild() { Schema s = SchemaBuilder .record("HandshakeRequest") .namespace("org.apache.avro.ipc").fields() .name("clientProtocol").type().optional().stringType() .name("meta").type().optional().map().values().bytesType() .endRecord(); String schemaString = s.toString(); int mid = schemaString.length() / 2; Schema parsedStringSchema = new org.apache.avro.Schema.Parser().parse(s.toString()); Schema parsedArrayOfStringSchema = new org.apache.avro.Schema.Parser().parse (schemaString.substring(0, mid), schemaString.substring(mid)); assertNotNull(parsedStringSchema); assertNotNull(parsedArrayOfStringSchema); assertEquals(parsedStringSchema.toString(), parsedArrayOfStringSchema.toString()); }
private Schema primitive(Schema.Type type, Schema bare) { // test creation of bare schema by name Schema bareByName = SchemaBuilder.builder().type(type.getName()); Assert.assertEquals(Schema.create(type), bareByName); Assert.assertEquals(bareByName, bare); // return a schema with custom prop set Schema p = Schema.create(type); p.addProp("p", "v"); return p; }
public static void createEmptyAvroStream(final OutputStream outStream) throws IOException { final FieldAssembler<Schema> builder = SchemaBuilder.record("NiFi_ExecuteSQL_Record").namespace("any.data").fields(); final Schema schema = builder.endRecord(); final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema); try (final DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter)) { dataFileWriter.create(schema, outStream); } }