@Test public void testReadUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidListSchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList expected = new RecordWithUUIDList(); expected.uuids = Arrays.asList(u1, u2); File test = write(uuidListSchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidListSchema), test).get(0)); }
@Test public void shouldSerializeArray() { shouldSerializeTypeCorrectly( SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build(), ImmutableList.of(1, 2, 3), org.apache.avro.SchemaBuilder.array().items( org.apache.avro.SchemaBuilder.builder() .unionOf().nullType().and().intType().endUnion()) ); }
final FieldAssembler<Schema> builder = SchemaBuilder.record(tableName).namespace("any.data").fields(); builder.name(columnName).type().unionOf().nullBuilder().endNull().and().booleanType().endUnion().noDefault(); break; builder.name(columnName).type().unionOf().nullBuilder().endNull().and().intType().endUnion().noDefault(); } else { builder.name(columnName).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault(); u -> u.type(decimal.addToSchema(SchemaBuilder.builder().bytesType()))); } else { addNullableField(builder, columnName, u -> u.stringType()); ? u.type(LogicalTypes.date().addToSchema(SchemaBuilder.builder().intType())) : u.stringType()); break; addNullableField(builder, columnName, u -> options.useLogicalTypes ? u.type(LogicalTypes.timeMillis().addToSchema(SchemaBuilder.builder().intType())) : u.stringType()); break; addNullableField(builder, columnName, u -> options.useLogicalTypes ? u.type(LogicalTypes.timestampMillis().addToSchema(SchemaBuilder.builder().longType())) : u.stringType()); break;
@Test public void testNames() { Schema r = SchemaBuilder.record("Rec").fields() .name("f0").type().fixed("org.foo.MyFixed").size(1).noDefault() .name("f1").type("org.foo.MyFixed").noDefault() .name("f2").type("org.foo.MyFixed", "").noDefault() .name("f3").type("org.foo.MyFixed", null).noDefault() .name("f4").type("org.foo.MyFixed", "ignorethis").noDefault() Schema f = SchemaBuilder.builder("").fixed("Foo").size(1); Assert.assertEquals(Schema.createFixed("Foo", null, null, 1), f); r = SchemaBuilder.record("Rec").namespace("org.foo").fields() .name("f0").type().fixed("MyFixed").size(1).noDefault() .name("f1").type("org.foo.MyFixed").noDefault() .name("f2").type("org.foo.MyFixed", "").noDefault() r = SchemaBuilder.record("Rec").namespace("org.rec").fields() .name("f0").type().fixed("MyFixed").namespace("org.foo").size(1).noDefault() .name("f1").type("org.foo.MyFixed").noDefault() .name("f2").type("org.foo.MyFixed", "").noDefault() SchemaBuilder.fixed("org.test.long").size(1); SchemaBuilder.fixed("long").namespace("org.test").size(1); SchemaBuilder.builder("org.test").fixed("long").size(1);
env.sendEventMap(makeMapEvent(null, new int[2]), "MyNWInfraTypeWithMapProp"); } else if (eventRepresentationEnum.isAvroEvent()) { GenericData.Record event = new GenericData.Record(record("name").fields() .optionalString("simple") .name("myarray").type(array().items().longType()).noDefault() .name("mymap").type(map().values().stringType()).noDefault() .endRecord()); event.put("myarray", Arrays.asList(0, 0));
public static File generateEnumTypes(String filename, int nrows, String[][] categories) throws IOException { assert categories.length == 2 : "Needs only 2 columns"; File parentDir = Files.createTempDir(); File f = new File(parentDir, filename); DatumWriter<GenericRecord> w = new GenericDatumWriter<GenericRecord>(); DataFileWriter<GenericRecord> dw = new DataFileWriter<GenericRecord>(w); Schema enumSchema1 = SchemaBuilder.enumeration("CEnum1").symbols(categories[0]); Schema enumSchema2 = SchemaBuilder.enumeration("CEnum2").symbols(categories[1]); Schema schema = SchemaBuilder.builder() .record("test_enum_types").fields() .name("CEnum").type(enumSchema1).noDefault() .name("CUEnum").type().optional().type(enumSchema2) .endRecord(); System.out.println(schema); int numOfCategories1 = categories[0].length; int numOfCategories2 = categories[1].length; try { dw.create(schema, f); for (int i = 0; i < nrows; i++) { GenericRecord gr = new GenericData.Record(schema); gr.put("CEnum", new GenericData.EnumSymbol(enumSchema1, categories[0][i % numOfCategories1])); gr.put("CUEnum", i % (numOfCategories2+1) == 0 ? null : new GenericData.EnumSymbol(enumSchema2, categories[1][i % numOfCategories2])); dw.append(gr); } return f; } finally { dw.close();; } }
@Test public void testConvertToAvroNull() { Assert.assertNull("Avro null", JsonUtil.convertToAvro(GenericData.get(), null, Schema.create(Schema.Type.NULL))); Assert.assertNull("Avro nullable long", JsonUtil.convertToAvro(GenericData.get(), null, SchemaBuilder.nullable().longType())); Assert.assertNull("Avro long", JsonUtil.convertToAvro(GenericData.get(), null, SchemaBuilder.builder().longType())); }
fieldAssembler = fieldAssembler.name(pom.getPredicate().getId().replaceAll("[^\\w]", "_")).type().unionOf().map().values().unionOf().map().values().stringType().and().stringType().and().nullType().endUnion().and().nullType().endUnion().noDefault(); fieldAssembler = fieldAssembler.name(pom.getPredicate().getId().replaceAll("[^\\w]", "_")).type().unionOf().map().values().unionOf().map().values(targetSchema).and().type(targetSchema).and().nullType().endUnion().and().nullType().endUnion().noDefault(); if(targetSchema == null) fieldAssembler = fieldAssembler.name(predicateShortHand).type().unionOf().array().items().stringType().and().stringType().and().nullType().endUnion().noDefault(); fieldAssembler = fieldAssembler.name(predicateShortHand).type().unionOf().array().items(targetSchema).and().type(targetSchema).and().nullType().endUnion().noDefault();
@Test public void testNamespaces() { Schema s1 = SchemaBuilder.record("myrecord") .namespace("org.example") .fields() .name("myint").type().intType().noDefault() .endRecord(); Schema s2 = SchemaBuilder.record("org.example.myrecord") .fields() .name("myint").type().intType().noDefault() .endRecord(); Schema s3 = SchemaBuilder.record("org.example.myrecord") .namespace("org.example2") .fields() .name("myint").type().intType().noDefault() .endRecord(); Schema s4 = SchemaBuilder.builder("org.example").record("myrecord") .fields() .name("myint").type().intType().noDefault()
@Test public void testArray() { Schema longSchema = Schema.create(Schema.Type.LONG); Schema expected = Schema.createArray(longSchema); Schema schema1 = SchemaBuilder.array().items().longType(); Assert.assertEquals(expected, schema1); Schema schema2 = SchemaBuilder.array().items(longSchema); Assert.assertEquals(expected, schema2); Schema schema3 = SchemaBuilder.array().prop("p", "v") .items().type("long"); expected.addProp("p", "v"); Assert.assertEquals(expected, schema3); }
@Test public void testMap() { Schema intSchema = Schema.create(Schema.Type.INT); Schema expected = Schema.createMap(intSchema); Schema schema1 = SchemaBuilder.map().values().intType(); Assert.assertEquals(expected, schema1); Schema schema2 = SchemaBuilder.map().values(intSchema); Assert.assertEquals(expected, schema2); Schema schema3 = SchemaBuilder.map().prop("p", "v") .values().type("int"); expected.addProp("p", "v"); Assert.assertEquals(expected, schema3); }
@Test public void testReadUUIDArray() throws IOException { Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); LogicalTypes.uuid().addToSchema( uuidArraySchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidArraySchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDArray expected = new RecordWithUUIDArray(); expected.uuids = new UUID[] {u1, u2}; File test = write(uuidArraySchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidArraySchema), test).get(0)); }
@Test(expected = IllegalArgumentException.class) public void testRejectsNonRecordSchemas() { final DatasetDescriptor desc = new DatasetDescriptor.Builder() .schema(SchemaBuilder.array().items().stringType()) .build(); new CSVFileReader<GenericData.Record>(localfs, csvFile, desc, DataModelUtil.accessor(GenericData.Record.class, desc.getSchema())); }
@Test public void testWriteUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); Schema stringArraySchema = SchemaBuilder.record("RecordWithUUIDArray") .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); stringArraySchema.getField("uuids").schema() .addProp(SpecificData.CLASS_PROP, List.class.getName()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord expected = new GenericData.Record(stringArraySchema); expected.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList r = new RecordWithUUIDList(); r.uuids = Arrays.asList(u1, u2); File test = write(REFLECT, uuidListSchema, r); Assert.assertEquals("Should read UUIDs as Strings", expected, read(REFLECT.createDatumReader(stringArraySchema), test).get(0)); }