private static List<Schema.Field> _cloneFieldsAndResetPosition( List<Schema.Field> fields) { List<Schema.Field> copyFieldList = new ArrayList<>(); for (Schema.Field schemaEntry : fields) { Schema.Field field = new Schema.Field( schemaEntry.name(), schemaEntry.schema(), schemaEntry.doc(), schemaEntry.defaultVal(), schemaEntry.order()); Map<String, Object> objectProperties = schemaEntry.getObjectProps(); for (Map.Entry<String, Object> entry : objectProperties.entrySet()) { field.addProp(entry.getKey(), entry.getValue()); } copyFieldList.add(field); } return copyFieldList; }
for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), copyDocs ? field.doc() : null, field.defaultVal(), field.order()); copyProperties.copy(field, newField); newFields.add(newField);
private static void _copyFields( List<Schema.Field> fields, List<Schema.Field> copiedFieldList) { for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field( field.name(), field.schema(), field.doc(), field.defaultVal()); _copyFieldProperties(field.getObjectProps(), newField); copiedFieldList.add(newField); } }
public static List<Field> getSchemaFieldsList(Schema schema) { List<Field> result = new ArrayList<>(); for (Field f : schema.getFields()) { Field nf = new Field(f.name(), f.schema(), f.doc(), f.defaultVal()); nf.getObjectProps().putAll(f.getObjectProps()); for (Map.Entry<String, Object> entry : f.getObjectProps().entrySet()) { nf.addProp(entry.getKey(), entry.getValue()); } result.add(nf); } return result; }
private Field getMigratedField(Field origin, Schema expectedSchema, String expectedDIType) { Field expectedField = new Schema.Field(origin.name(), expectedSchema, origin.doc(), origin.defaultVal(), origin.order()); for (Map.Entry<String, Object> entry : origin.getObjectProps().entrySet()) { if ("di.column.talendType".equals(entry.getKey())) { expectedField.addProp("di.column.talendType", expectedDIType); } else { expectedField.addProp(entry.getKey(), entry.getValue()); } } return expectedField; }
public static Field generateNewField(Field origin) { Schema.Field field = new Schema.Field(origin.name(), origin.schema(), origin.doc(), origin.defaultVal(), origin.order()); field.getObjectProps().putAll(origin.getObjectProps()); for (Map.Entry<String, Object> entry : origin.getObjectProps().entrySet()) { field.addProp(entry.getKey(), entry.getValue()); } return field; }
static Schema.Field copyField(Schema.Field field, Schema newSchema, String newName) { Schema.Field copy = new Schema.Field(newName, newSchema, field.doc(), field.defaultVal(), field.order()); for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) { copy.addProp(prop.getKey(), prop.getValue()); } if (!newName.equals(field.name())) { copy.addAlias(field.name()); } return copy; } }
private static Schema.Field copyField(Schema.Field field, Schema newSchema) { Schema.Field copy = new Schema.Field(field.name(), newSchema, field.doc(), field.defaultVal(), field.order()); for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) { copy.addProp(prop.getKey(), prop.getValue()); } return copy; } }
private static Schema.Field copyField(Schema.Field field, Schema newSchema) { Schema.Field copy = new Schema.Field(field.name(), newSchema, field.doc(), field.defaultVal(), field.order()); for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) { copy.addProp(prop.getKey(), prop.getValue()); } return copy; } }
String fname = rf.name(); if (writer.getField(fname) == null) { if (rf.defaultVal() == null) { result = Symbol.error("Found " + writer.getFullName() + ", expecting " + reader.getFullName()
for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), field.doc(), field.defaultVal(), field.order()); copyAllProperties(field, newField); newFields.add(newField);
/** * Validates that a particular value for a given field is valid according to * the following algorithm: * 1. If the value is not null, or the field type is null, or the field type * is a union which accepts nulls, returns. * 2. Else, if the field has a default value, returns. * 3. Otherwise throws AvroRuntimeException. * @param field the field to validate. * @param value the value to validate. * @throws NullPointerException if value is null and the given field does * not accept null values. */ protected void validate(Field field, Object value) { if (isValidValue(field, value)) { return; } else if (field.defaultVal() != null) { return; } else { throw new AvroRuntimeException( "Field " + field + " does not accept null values"); } }
public static GenericRecord createAvroRecord(final Record record, final Schema avroSchema, final Charset charset) throws IOException { final GenericRecord rec = new GenericData.Record(avroSchema); final RecordSchema recordSchema = record.getSchema(); for (final RecordField recordField : recordSchema.getFields()) { final Object rawValue = record.getValue(recordField); Pair<String, Field> fieldPair = lookupField(avroSchema, recordField); final String fieldName = fieldPair.getLeft(); final Field field = fieldPair.getRight(); if (field == null) { continue; } final Object converted = convertToAvroObject(rawValue, field.schema(), fieldName, charset); rec.put(fieldName, converted); } // see if the Avro schema has any fields that aren't in the RecordSchema, and if those fields have a default // value then we want to populate it in the GenericRecord being produced for (final Field field : avroSchema.getFields()) { final Optional<RecordField> recordField = recordSchema.getField(field.name()); if (!recordField.isPresent() && rec.get(field.name()) == null && field.defaultVal() != null) { rec.put(field.name(), field.defaultVal()); } } return rec; }
/** * Adds fields to <tt>recordFields</tt> list. * @param recordFields - record fields are added to this list. * @param field - the field * @param fieldName - field name * @param fieldSchema - field schema * @param dataType - data type * @param nullable - is nullable? */ private static void addFieldToList(final List<RecordField> recordFields, final Field field, final String fieldName, final Schema fieldSchema, final DataType dataType, final boolean nullable) { if (field.defaultVal() == JsonProperties.NULL_VALUE) { recordFields.add(new RecordField(fieldName, dataType, field.aliases(), nullable)); } else { Object defaultValue = field.defaultVal(); if (fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) { defaultValue = defaultValue instanceof List ? ((List<?>) defaultValue).toArray() : new Object[0]; } recordFields.add(new RecordField(fieldName, dataType, defaultValue, field.aliases(), nullable)); } }
@Test public void testSelfReferenceInThreeUnion(){ Schema schema = new Schema.Parser().parse(THREE_TYPE_UNION); Field leftField = schema.getField("left"); assertEquals(JsonProperties.NULL_VALUE,leftField.defaultVal()); final Schema leftFieldSchema = leftField.schema(); assertEquals(Type.UNION,leftFieldSchema.getType()); assertEquals("null",leftFieldSchema.getTypes().get(0).getName()); assertEquals("string",leftFieldSchema.getTypes().get(1).getName()); assertEquals("Node",leftFieldSchema.getTypes().get(2).getName()); Field rightField = schema.getField("right"); assertEquals(JsonProperties.NULL_VALUE,rightField.defaultVal()); final Schema rightFieldSchema = rightField.schema(); assertEquals(Type.UNION,rightFieldSchema.getType()); assertEquals("null",rightFieldSchema.getTypes().get(0).getName()); assertEquals("string",rightFieldSchema.getTypes().get(1).getName()); assertEquals("Node",rightFieldSchema.getTypes().get(2).getName()); }
@Test public void testDefaultTypes() { Integer intDef = 1; Long longDef = 2L; Float floatDef = 3F; Double doubleDef = 4D; Schema schema = SchemaBuilder.record("r").fields() .name("int").type().intType().intDefault(intDef) .name("long").type().longType().longDefault(longDef) .name("float").type().floatType().floatDefault(floatDef) .name("double").type().doubleType().doubleDefault(doubleDef) .endRecord(); Assert.assertEquals("int field default type or value mismatch", intDef, schema.getField("int").defaultVal()); Assert.assertEquals("long field default type or value mismatch", longDef, schema.getField("long").defaultVal()); Assert.assertEquals("float field default type or value mismatch", floatDef, schema.getField("float").defaultVal()); Assert.assertEquals("double field default type or value mismatch", doubleDef, schema.getField("double").defaultVal()); }
@Test public void testSelfReferenceInUnion(){ Schema schema = new Schema.Parser().parse(SIMPLE_BINARY_TREE); Field leftField = schema.getField("left"); assertEquals(JsonProperties.NULL_VALUE,leftField.defaultVal()); final Schema leftFieldSchema = leftField.schema(); assertEquals(Type.UNION,leftFieldSchema.getType()); assertEquals("null",leftFieldSchema.getTypes().get(0).getName()); assertEquals("Node",leftFieldSchema.getTypes().get(1).getName()); Field rightField = schema.getField("right"); assertEquals(JsonProperties.NULL_VALUE,rightField.defaultVal()); final Schema rightFieldSchema = rightField.schema(); assertEquals(Type.UNION,rightFieldSchema.getType()); assertEquals("null",rightFieldSchema.getTypes().get(0).getName()); assertEquals("Node",rightFieldSchema.getTypes().get(1).getName()); }
@Test public void testR11() throws Exception { Schema r11Record = ReflectData.get().getSchema(R11.class); assertEquals(Schema.Type.RECORD, r11Record.getType()); Field r11Field = r11Record.getField("text"); assertEquals(JsonProperties.NULL_VALUE, r11Field.defaultVal()); Schema r11FieldSchema = r11Field.schema(); assertEquals(Schema.Type.UNION, r11FieldSchema.getType()); assertEquals(Schema.Type.NULL, r11FieldSchema.getTypes().get(0).getType()); Schema r11String = r11FieldSchema.getTypes().get(1); assertEquals(Schema.Type.STRING, r11String.getType()); R11 r11 = new R11(); checkReadWrite(r11, r11Record); r11.text = "foo"; checkReadWrite(r11, r11Record); }
@Test public void testFixedDefaultValueDrop() { Schema md5 = SchemaBuilder.builder().fixed("MD5").size(16); Schema frec = SchemaBuilder.builder().record("test") .fields().name("hash").type(md5).withDefault(new byte[16]).endRecord(); Schema.Field field = frec.getField("hash"); Assert.assertNotNull(field.defaultVal()); Assert.assertArrayEquals(new byte[16], (byte[]) field.defaultVal()); }
private GenericRecord createRecordWithDefaultField(Schema schema) { GenericRecord record = new GenericData.Record(schema); record.put("f", schema.getField("f").defaultVal()); return record; } }