private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
public static Optional<Schema> getKeySchemaFromRecord(Schema record) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = getKeySchema(field); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } if (!fields.isEmpty()) { Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getName(), false); newSchema.setFields(fields); return Optional.of(newSchema); } else { return Optional.absent(); } }
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { Preconditions.checkArgument(inputSchema.getFields().equals(gobblinTrackingEventSchema.getFields())); Schema outputSchema = Schema .createRecord(ConfigUtils.getString(config, NEW_SCHEMA_NAME, inputSchema.getName()), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(newFields); return outputSchema; }
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { List<Field> outputSchemaFields = new ArrayList<>(); for (Field field : inputSchema.getFields()) { outputSchemaFields.add(convertFieldSchema(inputSchema, field, workUnit)); } Schema outputSchema = Schema .createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(outputSchemaFields); return outputSchema; }
/*** * Flatten Record schema * @param schema Record Schema to flatten * @param shouldPopulateLineage If lineage information should be tagged in the field, this is true when we are * un-nesting fields * @param flattenComplexTypes Flatten complex types recursively other than Record and Option * @return Flattened Record Schema */ private Schema flattenRecord(Schema schema, boolean shouldPopulateLineage, boolean flattenComplexTypes) { Preconditions.checkNotNull(schema); Preconditions.checkArgument(Schema.Type.RECORD.equals(schema.getType())); Schema flattenedSchema; List<Schema.Field> flattenedFields = new ArrayList<>(); if (schema.getFields().size() > 0) { for (Schema.Field oldField : schema.getFields()) { List<Schema.Field> newFields = flattenField(oldField, ImmutableList.<String>of(), shouldPopulateLineage, flattenComplexTypes, Optional.<Schema>absent()); if (null != newFields && newFields.size() > 0) { flattenedFields.addAll(newFields); } } } flattenedSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); flattenedSchema.setFields(flattenedFields); return flattenedSchema; }
@Override public SchemaVisitorAction visitNonTerminal(final Schema nt) { Schema.Type type = nt.getType(); if (type == RECORD) { Schema newSchema = Schema.createRecord(nt.getName(), copyDocs ? nt.getDoc() : null, nt.getNamespace(), nt.isError()); copyProperties.copy(nt, newSchema); replace.put(nt, newSchema); } return SchemaVisitorAction.CONTINUE; }
public static Schema newSchema( Schema metadataSchema, String newSchemaName, List<Schema.Field> moreFields, int insertPoint) { Schema newSchema = Schema.createRecord( newSchemaName, metadataSchema.getDoc(), metadataSchema.getNamespace(), metadataSchema.isError()); List<Schema.Field> fields = metadataSchema.getFields(); List<Schema.Field> copyFieldList = _cloneFieldsAndResetPosition(fields); copyFieldList.addAll(insertPoint, moreFields); newSchema.setFields(copyFieldList); Map<String, Object> objectProperties = metadataSchema.getObjectProps(); for (Map.Entry<String, Object> entry : objectProperties.entrySet()) { newSchema.addProp(entry.getKey(), entry.getValue()); } return newSchema; }
/** * Copies the input {@link org.apache.avro.Schema} but changes the schema name. * @param schema {@link org.apache.avro.Schema} to copy. * @param newName name for the copied {@link org.apache.avro.Schema}. * @return A {@link org.apache.avro.Schema} that is a copy of schema, but has the name newName. */ public static Schema switchName(Schema schema, String newName) { if (schema.getName().equals(newName)) { return schema; } Schema newSchema = Schema.createRecord(newName, schema.getDoc(), schema.getNamespace(), schema.isError()); List<Field> fields = schema.getFields(); Iterable<Field> fieldsNew = Iterables.transform(fields, new Function<Field, Field>() { @Override public Schema.Field apply(Field input) { //this should never happen but the API has marked input as Nullable if (null == input) { return null; } Field field = new Field(input.name(), input.schema(), input.doc(), input.defaultValue(), input.order()); return field; } }); newSchema.setFields(Lists.newArrayList(fieldsNew)); return newSchema; }
private Schema removeFieldsFromRecords(Schema schema, Map<String, Schema> schemaMap) { Schema newRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); // Put an incomplete schema into schemaMap to avoid re-processing a recursive field. // The fields in the incomplete schema will be populated once the current schema is completely processed. schemaMap.put(schema.getFullName(), newRecord); List<Field> newFields = Lists.newArrayList(); for (Field field : schema.getFields()) { if (!this.shouldRemove(field)) { Field newField; if (this.children.containsKey(field.name())) { newField = new Field(field.name(), this.children.get(field.name()).removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } else { newField = new Field(field.name(), DO_NOTHING_INSTANCE.removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } newFields.add(newField); } } newRecord.setFields(newFields); return newRecord; }
@Override public Schema convertSchemaImpl(Schema inputSchema, WorkUnitState workUnitState) throws SchemaConversionException { if (inputSchema == null) { throw new SchemaConversionException("input schema is empty"); } List<Schema.Field> fields = Lists.newArrayList(); for (Schema.Field field : inputSchema.getFields()) { Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); fields.add(newField); } Schema.Field requestResponseField = new Schema.Field(HTTP_REQUEST_RESPONSE_FIELD, HttpRequestResponseRecord.getClassSchema(), "http output schema contains request url and return result", null); fields.add(requestResponseField); Schema combinedSchema = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc() + " (Http request and response are contained)", inputSchema.getNamespace(), false); combinedSchema.setFields(fields); return combinedSchema; }
@Override public SchemaVisitorAction visitNonTerminal(final Schema nt) { Schema.Type type = nt.getType(); if (type == Schema.Type.RECORD) { if (SchemaResolver.isUnresolvedSchema(nt)) { // unresolved schema will get a replacement that we already encountered, // or we will attempt to resolve. final String unresolvedSchemaName = SchemaResolver.getUnresolvedSchemaName(nt); Schema resSchema = symbolTable.apply(unresolvedSchemaName); if (resSchema == null) { throw new AvroTypeException("Unable to resolve " + unresolvedSchemaName); } Schema replacement = replace.get(resSchema); if (replacement == null) { replace.put(nt, Schemas.visit(resSchema, new ResolvingVisitor(resSchema, new IdentityHashMap<>(), symbolTable))); } else { replace.put(nt, replacement); } } else { // create a fieldless clone. Fields will be added in afterVisitNonTerminal. Schema newSchema = Schema.createRecord(nt.getName(), nt.getDoc(), nt.getNamespace(), nt.isError()); copyAllProperties(nt, newSchema); replace.put(nt, newSchema); } } return SchemaVisitorAction.CONTINUE; }
.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(fields); return outputSchema;
Schema.createRecord(newSchema.getName(), newSchema.getDoc(), newSchema.getNamespace(), newSchema.isError()); mergedSchema.setFields(combinedFields); return mergedSchema;
break; case ENUM: newSchema = Schema.createEnum(terminal.getName(), copyDocs ? terminal.getDoc() : null, terminal.getNamespace(), terminal.getEnumSymbols()); break; case FIXED: newSchema = Schema.createFixed(terminal.getName(), copyDocs ? terminal.getDoc() : null, terminal.getNamespace(), terminal.getFixedSize()); break;
Schema newRecord = Schema.createRecord(recordSchema.getName(), recordSchema.getDoc(), recordSchema.getNamespace(), recordSchema.isError()); newRecord.setFields(newFields); Schema newRecord = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); newRecord.setFields(newFields);
break; case ENUM: newSchema = Schema.createEnum(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), terminal.getEnumSymbols(), terminal.getEnumDefault()); break; case FIXED: newSchema = Schema.createFixed(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), terminal.getFixedSize()); break;
public static Schema appendFields( Schema schema, List<Schema.Field> fields) { if (schema.getType() != Schema.Type.RECORD) { TalendRuntimeExceptionBuilder talendRuntimeExceptionBuilder = TalendRuntimeException.build( CommonErrorCodes.UNEXPECTED_EXCEPTION); talendRuntimeExceptionBuilder.setAndThrow( "Schema type is not supported: " + schema.getType()); } Schema newSchema = Schema.createRecord( schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); List<Schema.Field> copiedFieldList = new ArrayList<>(); _copyFields(schema.getFields(), copiedFieldList); _copyFields(fields, copiedFieldList); newSchema.setFields(copiedFieldList); _copyFieldProperties(schema.getObjectProps(), newSchema); return newSchema; }
case RECORD: result = Schema.createRecord(s.getFullName(), s.getDoc(), null, s.isError()); for (String alias : s.getAliases())
@Test public void testDoc() { Schema s = SchemaBuilder.fixed("myfixed").doc("mydoc").size(1); Assert.assertEquals("mydoc", s.getDoc()); }
/** * Makes sure that "doc" tags are transcribed in the schemas. * Note that there are docs both for fields and for the records * themselves. */ @Test public void testDocs() { Schema schema = Schema.parse(SCHEMA_WITH_DOC_TAGS); assertEquals("This is not a world record.", schema.getDoc()); assertEquals("Inner Fixed", schema.getField("inner_fixed").doc()); assertEquals("Very Inner Fixed", schema.getField("inner_fixed").schema().getDoc()); assertEquals("Inner String", schema.getField("inner_string").doc()); assertEquals("Inner Enum", schema.getField("inner_enum").doc()); assertEquals("Very Inner Enum", schema.getField("inner_enum").schema().getDoc()); assertEquals("Inner Union", schema.getField("inner_union").doc()); }