/** * Flatten the <code>inputSchema</code> * {@inheritDoc} * @see gobblin.data.management.conversion.hive.converter.AbstractAvroToOrcConverter#convertSchema(org.apache.avro.Schema, gobblin.configuration.WorkUnitState) */ @Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) { return AVRO_FLATTENER.flatten(inputSchema, false); }
Schema flattenedFieldSchema = flatten(f.schema(), shouldPopulateLineage, flattenComplexTypes); if (shouldWrapInOption.isPresent()) { boolean isNullFirstMember = Schema.Type.NULL.equals(shouldWrapInOption.get().getTypes().get(0).getType());
flattenedSchema = Schema.createArray(flatten(schema.getElementType(), false)); } else { flattenedSchema = Schema.createArray(schema.getElementType()); case MAP: if (flattenComplexTypes) { flattenedSchema = Schema.createMap(flatten(schema.getValueType(), false)); } else { flattenedSchema = Schema.createMap(schema.getValueType());
/*** * Flatten the Schema to un-nest recursive Records (to make it optimal for ORC) * @param schema Avro Schema to flatten * @param flattenComplexTypes Flatten complex types recursively other than Record and Option * @return Flattened Avro Schema */ public Schema flatten(Schema schema, boolean flattenComplexTypes) { Preconditions.checkNotNull(schema); // To help make it configurable later this.flattenedNameJoiner = FLATTENED_NAME_JOINER; this.flattenedSourceJoiner = FLATTENED_SOURCE_JOINER; Schema flattenedSchema = flatten(schema, false, flattenComplexTypes); LOG.debug("Original Schema : " + schema); LOG.debug("Flattened Schema: " + flattenedSchema); return flattenedSchema; }
/*** * Flatten Union Schema * @param schema Union Schema to flatten * @param shouldPopulateLineage If lineage information should be tagged in the field, this is true when we are * un-nesting fields * @param flattenComplexTypes Flatten complex types recursively other than Record and Option * @return Flattened Union Schema */ private Schema flattenUnion(Schema schema, boolean shouldPopulateLineage, boolean flattenComplexTypes) { Preconditions.checkNotNull(schema); Preconditions.checkArgument(Schema.Type.UNION.equals(schema.getType())); Schema flattenedSchema; List<Schema> flattenedUnionMembers = new ArrayList<>(); if (null != schema.getTypes() && schema.getTypes().size() > 0) { for (Schema oldUnionMember : schema.getTypes()) { if (flattenComplexTypes) { // It's member might still recursively contain records flattenedUnionMembers.add(flatten(oldUnionMember, shouldPopulateLineage, flattenComplexTypes)); } else { flattenedUnionMembers.add(oldUnionMember); } } } flattenedSchema = Schema.createUnion(flattenedUnionMembers); return flattenedSchema; }