@SuppressWarnings(value="unchecked") private Object generate(Schema schema, Random random, int d) { switch (schema.getType()) { case RECORD: GenericRecord record = new GenericData.Record(schema); for (Schema.Field field : schema.getFields()) { Object value = (field.getObjectProp(USE_DEFAULT) == null) ? generate(field.schema(), random, d+1) : GenericData.get().getDefaultValue(field); record.put(field.name(), value); List<String> symbols = schema.getEnumSymbols(); return new GenericData.EnumSymbol (schema, symbols.get(random.nextInt(symbols.size()))); new GenericData.Array(length<=0?0:length, schema); for (int i = 0; i < length; i++) array.add(generate(schema.getElementType(), random, d+1)); return array; case MAP:
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
@Override public Object getField(String fieldName) { Object value = record.get(fieldName); if (value instanceof Utf8) { return ((Utf8) value).toString(); } else if (value instanceof org.apache.avro.generic.GenericRecord) { org.apache.avro.generic.GenericRecord avroRecord = (org.apache.avro.generic.GenericRecord) value; org.apache.avro.Schema recordSchema = avroRecord.getSchema(); List<Field> fields = recordSchema.getFields() .stream() .map(f -> new Field(f.name(), f.pos())) .collect(Collectors.toList()); return new GenericAvroRecord(schema, fields, avroRecord); } else { return value; } }
@Test public void testReadUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidListSchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList expected = new RecordWithUUIDList(); expected.uuids = Arrays.asList(u1, u2); File test = write(uuidListSchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidListSchema), test).get(0)); }
public static void fillComplexFullResyncDelta(GenericRecord delta) { GenericRecord testField2 = new GenericData.Record(getSchemaByFullName( delta.getSchema().getField("testField2").schema().getTypes(), "org.kaa.config.testRecordT")); testField2.put("testField3", 456); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema() .getField("__uuid").schema(), rawUuid); delta.put("testField1", "abc"); delta.put("testField2", testField2); delta.put("__uuid", uuid); }
Schema avroSchema = schemaParser.parse(configuraionSchema.getRawSchema()); GenericRecord recordNew1 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordNew1.put("intField", 4); GenericFixed uuidNew1 = AvroUtils.generateUuidObject(); recordNew1.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord recordNew2 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordNew2.put("intField", 5); recordNew2.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord rootNew = new GenericData.Record(avroSchema); GenericArray arrayNew = new GenericData.Array<>(2, rootNew.getSchema().getField("complexArrayField").schema()); arrayNew.add(recordNew1); arrayNew.add(recordNew2); rootNew.put("complexArrayField", arrayNew); GenericRecord rootOld = new GenericData.Record(avroSchema); GenericArray arrayOld = new GenericData.Array<>(2, rootOld.getSchema().getField("complexArrayField").schema()); arrayOld.add(recordOld1); arrayOld.add(recordOld2); rootOld.put("complexArrayField", arrayOld);
@Test public void testByteBufferDeepCopy() { // Test that a deep copy of a byte buffer respects the byte buffer // limits and capacity. byte[] buffer_value = {0, 1, 2, 3, 0, 0, 0}; ByteBuffer buffer = ByteBuffer.wrap(buffer_value, 1, 4); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); Field byte_field = new Field("bytes", Schema.create(Type.BYTES), null, null); schema.setFields(Arrays.asList(byte_field)); GenericRecord record = new GenericData.Record(schema); record.put(byte_field.name(), buffer); GenericRecord copy = GenericData.get().deepCopy(schema, record); ByteBuffer buffer_copy = (ByteBuffer) copy.get(byte_field.name()); assertEquals(buffer, buffer_copy); }
Schema rootSchema = root.getSchema(); for (Field field : rootSchema.getFields()) { Object value = root.get(field.name()); if (value instanceof List) { List<Object> values = (List<Object>) value; Schema arraySchema = getArraySchema(delta, field.name()); GenericArray deltaArray = new GenericData.Array(values.size(), arraySchema); for (Object item : values) { addComplexItemToArray(record, deltaArray); } else { deltaArray.add(item); delta.put(field.name(), deltaArray); } else if (value instanceof GenericContainer) { processComplexField(delta, field.name(), (GenericContainer) value, null, null); } else { delta.put(field.name(), value);
@Test public void testRecursiveRecord() { Schema schema = SchemaBuilder.record("LongList").fields() .name("value").type().longType().noDefault() .name("next").type().optional().type("LongList") .endRecord(); Assert.assertEquals("LongList", schema.getName()); List<Schema.Field> fields = schema.getFields(); Assert.assertEquals(2, fields.size()); Assert.assertEquals( new Schema.Field("value", Schema.create(Schema.Type.LONG), null, null), fields.get(0)); Assert.assertEquals( Schema.Type.UNION, fields.get(1).schema().getType()); Assert.assertEquals( Schema.Type.NULL, fields.get(1).schema().getTypes().get(0).getType()); Schema recordSchema = fields.get(1).schema().getTypes().get(1); Assert.assertEquals(Schema.Type.RECORD, recordSchema.getType()); Assert.assertEquals("LongList", recordSchema.getName()); Assert.assertEquals(NullNode.getInstance(), fields.get(1).defaultValue()); }
String newStr = new String(baosNew.toByteArray(), "UTF-8"); BaseData oldData = new BaseData(new BaseSchema(schema.toString()), oldStr); BaseData newData = new BaseData(new BaseSchema(schema.toString()), newStr); RawBinaryDelta deltaResult = calculator.calculate(oldData, newData); GenericRecord delta1 = new GenericData.Record(getDeltaSchemaByFullName(protocolSchema, "org.kaa.config.testT")); GenericEnumSymbol reset = new GenericData.EnumSymbol(getSchemaByFullName(delta1.getSchema().getField("testField1").schema().getTypes(), "org.kaaproject.configuration.resetT"), "reset"); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta1.getSchema().getField("__uuid").schema(), rawUuid); delta1.put("testField1", reset); delta1.put("__uuid", uuid); GenericRecord delta2 = new GenericData.Record(getDeltaSchemaByFullName(protocolSchema, "org.kaa.config.testT")); delta2.put("__uuid", uuid); GenericArray<Integer> testField1 = new GenericData.Array<Integer>(3, getArraySchema(delta2, "testField1")); testField1.add(321); testField1.add(456); testField1.add(654); delta2.put("testField1", testField1);
@SuppressWarnings("unchecked") @Test public void testRemoveUuid() { Schema uuidSchema = Schema.createFixed("uuidT", "", "org.kaaproject.configuration", 16); Schema.Field uuidField = new Schema.Field("__uuid", uuidSchema, "", null); Schema recordSchema = Schema.createRecord(Arrays.asList(uuidField)); Assert.assertNull("Uuid is still present after removal in recordWithoutUuid", recordWithUuid.get("__uuid")); Schema otherFixedSchema = Schema.createFixed("__uuid", "", "not.a", 1); Schema.Field otherFixed = new Schema.Field("some_field", otherFixedSchema, "", null); byte[] otherFixedValue = {1}; Schema.Field innerRecordField = new Schema.Field("inner", recordSchema, "", null); Schema.Field intField = new Schema.Field("int_value", intFieldSchema, "", null); Schema.Field uuidInArrayField = new Schema.Field("__uuid", uuidSchema, "", null); GenericRecord inArrayRecord1 = new GenericData.Record(inArrayRecordSchema); inArrayRecord1.put("__uuid", GenericData.get().createFixed(null, uuid_value, uuidSchema)); GenericRecord inArrayRecord2 = new GenericData.Record(inArrayRecordSchema); Schema.Field arrayField = new Schema.Field("array1", arraySchema, "", null); GenericArray<GenericRecord> array1 = new GenericData.Array<GenericRecord>(2, arraySchema); array1.add(inArrayRecord1); array1.add(inArrayRecord2); Schema.Field uuidOfComplexRecordField = new Schema.Field("__uuid", uuidSchema, "", null);
public static void fillArrayFullResyncDelta(GenericRecord delta) { byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema() .getField("__uuid").schema(), rawUuid); delta.put("__uuid", uuid); GenericRecord itemRecord1 = new GenericData.Record(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaa.config.testRecordItemT")); itemRecord1.put("testField2", 1); byte[] rawItemUuid1 = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; GenericFixed itemUuid1 = new GenericData.Fixed(itemRecord1 .getSchema().getField("__uuid").schema(), rawItemUuid1); itemRecord1.put("__uuid", itemUuid1); GenericRecord itemRecord2 = new GenericData.Record(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaa.config.testRecordItemT")); itemRecord2.put("testField2", 2); byte[] rawItemUuid2 = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; GenericFixed itemUuid2 = new GenericData.Fixed(itemRecord2 .getSchema().getField("__uuid").schema(), rawItemUuid2); itemRecord2.put("__uuid", itemUuid2); GenericRecord itemRecord3 = new GenericData.Record(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaa.config.testRecordItemT")); itemRecord3.put("testField2", 3); byte[] rawItemUuid3 = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}; GenericFixed itemUuid3 = new GenericData.Fixed(itemRecord3
@SuppressWarnings("unchecked") @Test public void testCanonizeRecord() { Schema arrayASchema = Schema.createArray(Schema.create(Type.INT)); Schema arrayBSchema = Schema.createArray(Schema.create(Type.STRING)); Schema.Field arrayAField = new Schema.Field("arrayA", arrayASchema, "", null); Schema.Field arrayBField = new Schema.Field("arrayB", arrayBSchema, "", null); Schema recordASchema = Schema.createRecord(Arrays.asList(arrayBField)); Schema.Field recordAField = new Schema.Field("recordA", recordASchema, "", null); Schema rootRecordSchema = Schema.createRecord(Arrays.asList(arrayAField, recordAField)); GenericRecord mixedRecord = new GenericData.Record(rootRecordSchema); arrayA.add(9); arrayA.add(8); arrayA.add(2); arrayA.add(1); mixedRecord.put("arrayA", arrayA); GenericRecord recordA = new GenericData.Record(recordASchema); GenericArray<String> arrayB = new GenericData.Array<String>(5, arrayBSchema); arrayB.add("abc"); arrayB.add("ghi"); arrayB.add("def"); recordA.put("arrayB", arrayB); mixedRecord.put("recordA", recordA);
@Test public void testToStringSameValues() throws IOException { List<Field> fields = new ArrayList<>(); fields.add(new Field("nullstring1", Schema.create(Type.STRING), null, (Object)null)); fields.add(new Field("nullstring2", Schema.create(Type.STRING), null, (Object)null)); fields.add(new Field("string1", Schema.create(Type.STRING ), null, (Object)null)); fields.add(new Field("string2", Schema.create(Type.STRING ), null, (Object)null)); fields.add(new Field("bytes1", Schema.create(Type.BYTES ), null, (Object)null)); fields.add(new Field("bytes2", Schema.create(Type.BYTES ), null, (Object)null)); fields.add(new Field("int1", Schema.create(Type.INT ), null, (Object)null)); fields.add(new Field("int2", Schema.create(Type.INT ), null, (Object)null)); fields.add(new Field("long1", Schema.create(Type.LONG ), null, (Object)null)); fields.add(new Field("long2", Schema.create(Type.LONG ), null, (Object)null)); fields.add(new Field("float1", Schema.create(Type.FLOAT ), null, (Object)null)); fields.add(new Field("float2", Schema.create(Type.FLOAT ), null, (Object)null)); fields.add(new Field("double1", Schema.create(Type.DOUBLE ), null, (Object)null)); fields.add(new Field("double2", Schema.create(Type.DOUBLE ), null, (Object)null)); fields.add(new Field("boolean1",Schema.create(Type.BOOLEAN ), null, (Object)null)); fields.add(new Field("boolean2",Schema.create(Type.BOOLEAN ), null, (Object)null)); fields.add(new Field("enum1", enumSchema, null, (Object)null)); fields.add(new Field("enum2", enumSchema, null, (Object)null));
@Test public void testP2() throws Exception { Schema e1 = ReflectData.get().getSchema(E1.class); assertEquals(Schema.Type.RECORD, e1.getType()); assertTrue(e1.isError()); Field message = e1.getField("detailMessage"); assertNotNull("field 'detailMessage' should not be null", message); Schema messageSchema = message.schema(); assertEquals(Schema.Type.UNION, messageSchema.getType()); assertEquals(Schema.Type.NULL, messageSchema.getTypes().get(0).getType()); assertEquals(Schema.Type.STRING, messageSchema.getTypes().get(1).getType()); Protocol p2 = ReflectData.get().getProtocol(P2.class); Protocol.Message m = p2.getMessages().get("error"); // check error schema is union Schema response = m.getErrors(); assertEquals(Schema.Type.UNION, response.getType()); assertEquals(Schema.Type.STRING, response.getTypes().get(0).getType()); assertEquals(e1, response.getTypes().get(1)); }
@Test public void testR11() throws Exception { Schema r11Record = ReflectData.get().getSchema(R11.class); assertEquals(Schema.Type.RECORD, r11Record.getType()); Field r11Field = r11Record.getField("text"); assertEquals(JsonProperties.NULL_VALUE, r11Field.defaultVal()); Schema r11FieldSchema = r11Field.schema(); assertEquals(Schema.Type.UNION, r11FieldSchema.getType()); assertEquals(Schema.Type.NULL, r11FieldSchema.getTypes().get(0).getType()); Schema r11String = r11FieldSchema.getTypes().get(1); assertEquals(Schema.Type.STRING, r11String.getType()); R11 r11 = new R11(); checkReadWrite(r11, r11Record); r11.text = "foo"; checkReadWrite(r11, r11Record); }
@Test public void testSwitchNamespace() { String originalNamespace = "originalNamespace"; String originalName = "originalName"; String newNamespace = "newNamespace"; Schema schema = SchemaBuilder.builder(originalNamespace).record(originalName).fields(). requiredDouble("double").optionalFloat("float").endRecord(); Map<String, String> map = Maps.newHashMap(); map.put(originalNamespace, newNamespace); Schema newSchema = AvroUtils.switchNamespace(schema, map); Assert.assertEquals(newSchema.getNamespace(), newNamespace); Assert.assertEquals(newSchema.getName(), originalName); for(Schema.Field field : newSchema.getFields()) { Assert.assertEquals(field, schema.getField(field.name())); } }
/** * Creates a GenericRecordBuilder by copying an existing record instance. * @param other the record instance to copy. */ public GenericRecordBuilder(Record other) { super(other.getSchema(), GenericData.get()); record = new GenericData.Record(other, /* deepCopy = */ true); // Set all fields in the RecordBuilder that are set in the record for (Field f : schema().getFields()) { Object value = other.get(f.pos()); // Only set the value if it is not null, if the schema type is null, // or if the schema type is a union that accepts nulls. if (isValidValue(f, value)) { set(f, data().deepCopy(f.schema(), value)); } } }