private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
/** * Makes sure the default value is good * * @param parent * @param field */ /* package private */static void checkDefaultValueIsLegal(Field field, List<Message> messages, String name) { if(field == null) { throw new IllegalArgumentException("Field must be non-null. Name=" + name); } if(field.defaultValue() != null) { // Get the type schema. If this is a UNION, the default must be of // the leading type Schema fieldSchema = field.schema(); if(fieldSchema.getType() == Schema.Type.UNION) { fieldSchema = fieldSchema.getTypes().get(0); } // Get the default value JsonNode defaultJson = field.defaultValue(); String expectedVal = checkDefaultJson(defaultJson, field.schema()); if(expectedVal != null) { messages.add(new Message(Level.ERROR, "Illegal default value for field " + name + ". The default must be of type " + expectedVal + ".")); } } }
public static Optional<Schema> getKeySchemaFromRecord(Schema record) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = getKeySchema(field); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } if (!fields.isEmpty()) { Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getName(), false); newSchema.setFields(fields); return Optional.of(newSchema); } else { return Optional.absent(); } }
defaultValue.has(field.name()) ? defaultValue.get(field.name()) : field.defaultValue())) return false; return true;
fields.add(new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order())); Field copy = new Field(name, field.schema(), field.doc(), field.defaultValue(), field.order()); fields.add(copy);
combinedFields.add(new Field(newFld.name(), newFld.schema(), newFld.doc(), newFld.defaultValue())); combinedFields.add(new Field(oldFld.name(), newFldSchema, oldFld.doc(), oldFld.defaultValue())); } else { union.add(Schema.create(Type.NULL)); union.add(oldFldSchema); Schema newFldSchema = Schema.createUnion(union); combinedFields.add(new Field(oldFld.name(), newFldSchema, oldFld.doc(), oldFld.defaultValue()));
JsonNode json = field.defaultValue(); if (json == null) throw new AvroRuntimeException("Field " + field
if (fieldsRenameMap.containsKey(curFieldName)) { newFields.add( new Schema.Field(fieldsRenameMap.get(curFieldName), field.schema(), field.doc(), field.defaultValue())); } else { newFields.add(new Schema.Field(curFieldName, field.schema(), field.doc(), field.defaultValue()));
if (readerField.defaultValue() == null) {
new Field(innerSrcField.name(), innerSrcField.schema(), innerSrcField.doc(), innerSrcField.defaultValue())); } else { Schema innerSrcSchema = innerSrcField.schema(); new Field(innerSrcField.name(), innerDestSchema, innerSrcField.doc(), innerSrcField.defaultValue()); newFields.add(innerDestField);
void fieldsToJson(Names names, JsonGenerator gen) throws IOException { gen.writeStartArray(); for (Field f : fields) { gen.writeStartObject(); gen.writeStringField("name", f.name()); gen.writeFieldName("type"); f.schema().toJson(names, gen); if (f.doc() != null) gen.writeStringField("doc", f.doc()); if (f.defaultValue() != null) { gen.writeFieldName("default"); gen.writeTree(f.defaultValue()); } if (f.order() != Field.Order.ASCENDING) gen.writeStringField("order", f.order().name); if (f.aliases != null && f.aliases.size() != 0) { gen.writeFieldName("aliases"); gen.writeStartArray(); for (String alias : f.aliases) gen.writeString(alias); gen.writeEndArray(); } f.writeProps(gen); gen.writeEndObject(); } gen.writeEndArray(); } }
void fieldsToJson(Names names, JsonGenerator gen) throws IOException { gen.writeStartArray(); for (Field f : fields) { gen.writeStartObject(); gen.writeStringField("name", f.name()); gen.writeFieldName("type"); f.schema().toJson(names, gen); if (f.doc() != null) gen.writeStringField("doc", f.doc()); if (f.defaultValue() != null) { gen.writeFieldName("default"); gen.writeTree(f.defaultValue()); } if (f.order() != Field.Order.ASCENDING) gen.writeStringField("order", f.order().name); if (f.aliases != null && f.aliases.size() != 0) { gen.writeFieldName("aliases"); gen.writeStartArray(); for (String alias : f.aliases) gen.writeString(alias); gen.writeEndArray(); } f.writeProps(gen); gen.writeEndObject(); } gen.writeEndArray(); } }
/** * Validates that a particular value for a given field is valid according to * the following algorithm: * 1. If the value is not null, or the field type is null, or the field type * is a union which accepts nulls, returns. * 2. Else, if the field has a default value, returns. * 3. Otherwise throws AvroRuntimeException. * @param field the field to validate. * @param value the value to validate. * @throws NullPointerException if value is null and the given field does * not accept null values. */ protected void validate(Field field, Object value) { if (isValidValue(field, value)) { return; } else if (field.defaultValue() != null) { return; } else { throw new AvroRuntimeException( "Field " + field + " does not accept null values"); } }
@Test public void testRecursiveRecord() { Schema schema = SchemaBuilder.record("LongList").fields() .name("value").type().longType().noDefault() .name("next").type().optional().type("LongList") .endRecord(); Assert.assertEquals("LongList", schema.getName()); List<Schema.Field> fields = schema.getFields(); Assert.assertEquals(2, fields.size()); Assert.assertEquals( new Schema.Field("value", Schema.create(Schema.Type.LONG), null, null), fields.get(0)); Assert.assertEquals( Schema.Type.UNION, fields.get(1).schema().getType()); Assert.assertEquals( Schema.Type.NULL, fields.get(1).schema().getTypes().get(0).getType()); Schema recordSchema = fields.get(1).schema().getTypes().get(1); Assert.assertEquals(Schema.Type.RECORD, recordSchema.getType()); Assert.assertEquals("LongList", recordSchema.getName()); Assert.assertEquals(NullNode.getInstance(), fields.get(1).defaultValue()); }
private Schema removeFieldsFromRecords(Schema schema, Map<String, Schema> schemaMap) { Schema newRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); // Put an incomplete schema into schemaMap to avoid re-processing a recursive field. // The fields in the incomplete schema will be populated once the current schema is completely processed. schemaMap.put(schema.getFullName(), newRecord); List<Field> newFields = Lists.newArrayList(); for (Field field : schema.getFields()) { if (!this.shouldRemove(field)) { Field newField; if (this.children.containsKey(field.name())) { newField = new Field(field.name(), this.children.get(field.name()).removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } else { newField = new Field(field.name(), DO_NOTHING_INSTANCE.removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } newFields.add(newField); } } newRecord.setFields(newFields); return newRecord; }
@Override public Schema convertSchemaImpl(Schema inputSchema, WorkUnitState workUnitState) throws SchemaConversionException { if (inputSchema == null) { throw new SchemaConversionException("input schema is empty"); } List<Schema.Field> fields = Lists.newArrayList(); for (Schema.Field field : inputSchema.getFields()) { Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); fields.add(newField); } Schema.Field requestResponseField = new Schema.Field(HTTP_REQUEST_RESPONSE_FIELD, HttpRequestResponseRecord.getClassSchema(), "http output schema contains request url and return result", null); fields.add(requestResponseField); Schema combinedSchema = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc() + " (Http request and response are contained)", inputSchema.getNamespace(), false); combinedSchema.setFields(fields); return combinedSchema; }
/** * Create a payload field with its latest schema fetched from {@link #registry} * * @param field the original payload field from input envelope schema * @return a new payload field with its latest schema */ private Field createLatestPayloadField(Field field) throws SchemaConversionException { try { Schema payloadSchema = fetchLatestPayloadSchema(); return new Field(field.name(), payloadSchema, DECORATED_PAYLOAD_DOC, field.defaultValue(), field.order()); } catch (Exception e) { throw new SchemaConversionException(e); } }
@Override protected JsonNode defaultValue(Field field) { return field.defaultValue(); }
@Override public Schema.Field apply(Field input) { //this should never happen but the API has marked input as Nullable if (null == input) { return null; } Field field = new Field(input.name(), input.schema(), input.doc(), input.defaultValue(), input.order()); return field; } });
/** * Convert to the output schema of a field */ protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit) throws SchemaConversionException { if (field.name().equals(payloadField)) { // Create a payload field with latest schema return createLatestPayloadField(field); } // Make a copy of the field to the output schema return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); }