@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { Preconditions.checkArgument(inputSchema.getFields().equals(gobblinTrackingEventSchema.getFields())); Schema outputSchema = Schema .createRecord(ConfigUtils.getString(config, NEW_SCHEMA_NAME, inputSchema.getName()), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(newFields); return outputSchema; }
@Override public GenericRecord partitionForRecord(GenericRecord record) { GenericRecord partition = new GenericData.Record(SCHEMA); partition.put(SCHEMA_STRING, record.getSchema().toString()); return partition; } }
@Nullable private LogicalType determineTimestampSpecLogicalType(Schema schema, String timestampSpecField) { for (Schema.Field field : schema.getFields()) { if (field.name().equals(timestampSpecField)) { return field.schema().getLogicalType(); } } return null; }
/* package private */Schema stripOptionalTypeUnion(Schema schema) { if(schema.getType() == Schema.Type.UNION && schema.getTypes().size() == 2 && schema.getTypes().contains(NULL_TYPE_SCHEMA)) { return schema.getTypes().get(0).equals(NULL_TYPE_SCHEMA) ? schema.getTypes().get(1) : schema.getTypes().get(0); } return schema; }
@Test public void testByteBufferDeepCopy() { // Test that a deep copy of a byte buffer respects the byte buffer // limits and capacity. byte[] buffer_value = {0, 1, 2, 3, 0, 0, 0}; ByteBuffer buffer = ByteBuffer.wrap(buffer_value, 1, 4); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); Field byte_field = new Field("bytes", Schema.create(Type.BYTES), null, null); schema.setFields(Arrays.asList(byte_field)); GenericRecord record = new GenericData.Record(schema); record.put(byte_field.name(), buffer); GenericRecord copy = GenericData.get().deepCopy(schema, record); ByteBuffer buffer_copy = (ByteBuffer) copy.get(byte_field.name()); assertEquals(buffer, buffer_copy); }
public static void fillComplexFullResyncDelta(GenericRecord delta) { GenericRecord testField2 = new GenericData.Record(getSchemaByFullName( delta.getSchema().getField("testField2").schema().getTypes(), "org.kaa.config.testRecordT")); testField2.put("testField3", 456); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema() .getField("__uuid").schema(), rawUuid); delta.put("testField1", "abc"); delta.put("testField2", testField2); delta.put("__uuid", uuid); }
@Test public void testNestedNullNamespaceReferencingWithUnion() { Schema inner = Schema.parse("{\"type\":\"record\",\"name\":\"Inner\",\"fields\":[]}"); Schema innerUnion = Schema.createUnion(Arrays.asList(inner, Schema.create(Type.NULL))); Schema outer = Schema.createRecord("Outer", null, "space", false); outer.setFields(Arrays.asList(new Field("f1", innerUnion, null, null), new Field("f2", innerUnion, null, null))); assertEquals(outer, Schema.parse(outer.toString())); }
@Test public void testReadUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidListSchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList expected = new RecordWithUUIDList(); expected.uuids = Arrays.asList(u1, u2); File test = write(uuidListSchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidListSchema), test).get(0)); }
@Test public void testR11() throws Exception { Schema r11Record = ReflectData.get().getSchema(R11.class); assertEquals(Schema.Type.RECORD, r11Record.getType()); Field r11Field = r11Record.getField("text"); assertEquals(JsonProperties.NULL_VALUE, r11Field.defaultVal()); Schema r11FieldSchema = r11Field.schema(); assertEquals(Schema.Type.UNION, r11FieldSchema.getType()); assertEquals(Schema.Type.NULL, r11FieldSchema.getTypes().get(0).getType()); Schema r11String = r11FieldSchema.getTypes().get(1); assertEquals(Schema.Type.STRING, r11String.getType()); R11 r11 = new R11(); checkReadWrite(r11, r11Record); r11.text = "foo"; checkReadWrite(r11, r11Record); }
@Test public void testSwitchNamespace() { String originalNamespace = "originalNamespace"; String originalName = "originalName"; String newNamespace = "newNamespace"; Schema schema = SchemaBuilder.builder(originalNamespace).record(originalName).fields(). requiredDouble("double").optionalFloat("float").endRecord(); Map<String, String> map = Maps.newHashMap(); map.put(originalNamespace, newNamespace); Schema newSchema = AvroUtils.switchNamespace(schema, map); Assert.assertEquals(newSchema.getNamespace(), newNamespace); Assert.assertEquals(newSchema.getName(), originalName); for(Schema.Field field : newSchema.getFields()) { Assert.assertEquals(field, schema.getField(field.name())); } }
/** * Creates a GenericRecordBuilder by copying an existing record instance. * @param other the record instance to copy. */ public GenericRecordBuilder(Record other) { super(other.getSchema(), GenericData.get()); record = new GenericData.Record(other, /* deepCopy = */ true); // Set all fields in the RecordBuilder that are set in the record for (Field f : schema().getFields()) { Object value = other.get(f.pos()); // Only set the value if it is not null, if the schema type is null, // or if the schema type is a union that accepts nulls. if (isValidValue(f, value)) { set(f, data().deepCopy(f.schema(), value)); } } }
public static boolean isOptionalPrimitive(Schema schema) { return schema.getType().equals(Schema.Type.UNION) && schema.getTypes().size() == 2 && ( (schema.getTypes().get(0).getType().equals(Schema.Type.NULL) && (isPrimitive(schema.getTypes().get(1)) || isPrimitiveArray(schema.getTypes().get(1)))) || (schema.getTypes().get(1).getType().equals(Schema.Type.NULL) && (isPrimitive(schema.getTypes().get(0)) || isPrimitiveArray(schema.getTypes().get(0)))) ); }
public static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
public static String generateHiveDDL(Schema avroSchema, String tableName) { Schema.Type schemaType = avroSchema.getType(); StringBuilder sb = new StringBuilder("CREATE EXTERNAL TABLE IF NOT EXISTS "); sb.append(tableName); sb.append(" ("); if (Schema.Type.RECORD.equals(schemaType)) { List<String> hiveColumns = new ArrayList<>(); List<Schema.Field> fields = avroSchema.getFields(); if (fields != null) { hiveColumns.addAll( fields.stream().map(field -> field.name() + " " + getHiveTypeFromAvroType(field.schema())).collect(Collectors.toList())); } sb.append(StringUtils.join(hiveColumns, ", ")); sb.append(") STORED AS ORC"); return sb.toString(); } else { throw new IllegalArgumentException("Avro schema is of type " + schemaType.getName() + ", not RECORD"); } }
private static TypeInfo generateRecordTypeInfo(Schema schema, Set<Schema> seenSchemas) throws AvroSerdeException { assert schema.getType().equals(Schema.Type.RECORD); if (seenSchemas == null) { seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>()); } else if (seenSchemas.contains(schema)) { throw new AvroSerdeException( "Recursive schemas are not supported. Recursive schema was " + schema .getFullName()); } seenSchemas.add(schema); List<Schema.Field> fields = schema.getFields(); List<String> fieldNames = new ArrayList<String>(fields.size()); List<TypeInfo> typeInfos = new ArrayList<TypeInfo>(fields.size()); for(int i = 0; i < fields.size(); i++) { fieldNames.add(i, fields.get(i).name()); typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); } return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); }
@Test public void testSwitchName() { String originalName = "originalName"; String newName = "newName"; Schema schema = SchemaBuilder.record(originalName).fields(). requiredDouble("double").optionalFloat("float").endRecord(); Schema newSchema = AvroUtils.switchName(schema, newName); Assert.assertEquals(newSchema.getName(), newName); for(Schema.Field field : newSchema.getFields()) { Assert.assertEquals(field, schema.getField(field.name())); } Assert.assertEquals(newName, AvroUtils.switchName(schema, newName).getName()); Assert.assertEquals(schema, AvroUtils.switchName(AvroUtils.switchName(schema, newName), schema.getName())); }
@Override public Iterable<GenericRecord> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { GenericRecord outputRecord = new GenericData.Record(outputSchema); for (Field field : inputRecord.getSchema().getFields()) { outputRecord.put(field.name(), convertFieldValue(outputSchema, field, inputRecord, workUnit)); } return new SingleRecordIterable<>(outputRecord); }