@Override public boolean equals(Object o) { if (!(o instanceof FieldType)) { return false; } FieldType other = (FieldType) o; return Objects.equals(getTypeName(), other.getTypeName()) && Objects.equals(getCollectionElementType(), other.getCollectionElementType()) && Objects.equals(getMapKeyType(), other.getMapKeyType()) && Objects.equals(getMapValueType(), other.getMapValueType()) && Objects.equals(getRowSchema(), other.getRowSchema()) && Arrays.equals(getMetadata(), other.getMetadata()); }
private String unparseRow(FieldType fieldType) { return "ROW<" + fieldType .getRowSchema() .getFields() .stream() .map(field -> field.getName() + " " + unparse(field.getType())) .collect(joining(",")) + ">"; } }
@Override public int hashCode() { return Arrays.deepHashCode( new Object[] { getTypeName(), getCollectionElementType(), getMapKeyType(), getMapValueType(), getRowSchema(), getMetadata() }); } }
static Row selectRow( Row input, FieldAccessDescriptor fieldAccessDescriptor, Schema inputSchema, Schema outputSchema) { if (fieldAccessDescriptor.allFields()) { return input; } else { Row.Builder output = Row.withSchema(outputSchema); for (int fieldId : fieldAccessDescriptor.fieldIdsAccessed()) { output.addValue(input.getValue(fieldId)); } for (Map.Entry<Integer, FieldAccessDescriptor> nested : fieldAccessDescriptor.nestedFields().entrySet()) { String fieldName = inputSchema.nameOf(nested.getKey()); Schema nestedInputSchema = inputSchema.getField(nested.getKey()).getType().getRowSchema(); Schema nestedOutputSchema = outputSchema.getField(fieldName).getType().getRowSchema(); output.addValue( selectRow( input.getValue(fieldName), nested.getValue(), nestedInputSchema, nestedOutputSchema)); } return output.build(); } } }
private static List<TableFieldSchema> toTableFieldSchema(Schema schema) { List<TableFieldSchema> fields = new ArrayList<>(schema.getFieldCount()); for (Field schemaField : schema.getFields()) { FieldType type = schemaField.getType(); TableFieldSchema field = new TableFieldSchema().setName(schemaField.getName()); if (schemaField.getDescription() != null && !"".equals(schemaField.getDescription())) { field.setDescription(schemaField.getDescription()); } if (!schemaField.getNullable()) { field.setMode(Mode.REQUIRED.toString()); } if (TypeName.ARRAY == type.getTypeName()) { type = type.getCollectionElementType(); field.setMode(Mode.REPEATED.toString()); } if (TypeName.ROW == type.getTypeName()) { Schema subType = type.getRowSchema(); field.setFields(toTableFieldSchema(subType)); } field.setType(toStandardSQLTypeName(type).toString()); fields.add(field); } return fields; }
private boolean equivalent(FieldType other) { if (!other.getTypeName().equals(getTypeName())) { return false; } switch (getTypeName()) { case ROW: if (!other.getRowSchema().equivalent(getRowSchema())) { return false; } break; case ARRAY: if (!other.getCollectionElementType().equivalent(getCollectionElementType())) { return false; } break; case MAP: if (!other.getMapKeyType().equivalent(getMapKeyType()) || !other.getMapValueType().equivalent(getMapValueType())) { return false; } break; default: return other.equals(this); } return true; }
static <T> T visit(SchemaZipFold<T> zipFold, Context context, FieldType left, FieldType right) { if (left.getTypeName() != right.getTypeName()) { return zipFold.accept(context, left, right); } Context newContext = context.withParent(left.getTypeName()); switch (left.getTypeName()) { case ARRAY: return zipFold.accumulate( zipFold.accept(context, left, right), visit( zipFold, newContext, left.getCollectionElementType(), right.getCollectionElementType())); case ROW: return visitRow(zipFold, newContext, left.getRowSchema(), right.getRowSchema()); case MAP: return zipFold.accumulate( zipFold.accept(context, left, right), visit( zipFold, newContext, left.getCollectionElementType(), right.getCollectionElementType())); default: return zipFold.accept(context, left, right); } }
/** Returns true if two FieldTypes are equal. */ public boolean typesEqual(FieldType other) { if (!Objects.equals(getTypeName(), other.getTypeName())) { return false; } if (!Arrays.equals(getMetadata(), other.getMetadata())) { return false; } if (getTypeName() == TypeName.ARRAY && !getCollectionElementType().typesEqual(other.getCollectionElementType())) { return false; } if (getTypeName() == TypeName.MAP && (!getMapValueType().typesEqual(other.getMapValueType()) || !getMapKeyType().typesEqual(other.getMapKeyType()))) { return false; } if (getTypeName() == TypeName.ROW && !getRowSchema().typesEqual(other.getRowSchema())) { return false; } return true; }
static void verifyFieldTypeSupported(Schema.FieldType fieldType) { Schema.TypeName fieldTypeName = fieldType.getTypeName(); if (fieldTypeName.isCompositeType()) { Schema rowFieldSchema = fieldType.getRowSchema(); rowFieldSchema.getFields().forEach(RowJsonValidation::verifyFieldTypeSupported); return; } if (fieldTypeName.isCollectionType()) { verifyFieldTypeSupported(fieldType.getCollectionElementType()); return; } if (!SUPPORTED_TYPES.contains(fieldTypeName)) { throw new RowJsonDeserializer.UnsupportedRowJsonException( fieldTypeName.name() + " is not supported when converting JSON objects to Rows. " + "Supported types are: " + SUPPORTED_TYPES.toString()); } } }
private static Schema getUnnestedSchema( Schema schema, List<String> nameComponents, SerializableFunction<List<String>, String> fn) { Schema.Builder builder = Schema.builder(); for (Field field : schema.getFields()) { nameComponents.add(field.getName()); if (field.getType().getTypeName().isCompositeType()) { Schema nestedSchema = getUnnestedSchema(field.getType().getRowSchema(), nameComponents, fn); for (Field nestedField : nestedSchema.getFields()) { builder.addField(nestedField); } } else { String name = fn.apply(nameComponents); Field newField = field.toBuilder().setName(name).build(); builder.addField(newField); } nameComponents.remove(nameComponents.size() - 1); } return builder.build(); } /** Unnest a row. */
@SuppressWarnings({"TypeParameterUnusedInFormals", "unchecked"}) private <T> T getValue(FieldType type, Object fieldValue, @Nullable Integer cacheKey) { if (type.getTypeName().equals(TypeName.ROW)) { return (T) new RowWithGetters(type.getRowSchema(), fieldValueGetterFactory, fieldValue); } else if (type.getTypeName().equals(TypeName.ARRAY)) { return cacheKey != null ? (T) cachedLists.computeIfAbsent( cacheKey, i -> getListValue(type.getCollectionElementType(), fieldValue)) : (T) getListValue(type.getCollectionElementType(), fieldValue); } else if (type.getTypeName().equals(TypeName.MAP)) { Map map = (Map) fieldValue; return cacheKey != null ? (T) cachedMaps.computeIfAbsent( cacheKey, i -> getMapValue(type.getMapKeyType(), type.getMapValueType(), map)) : (T) getMapValue(type.getMapKeyType(), type.getMapValueType(), map); } else { return (T) fieldValue; } }
static Schema getOutputSchema(Schema inputSchema, FieldAccessDescriptor fieldAccessDescriptor) { if (fieldAccessDescriptor.allFields()) { return inputSchema; } Schema.Builder builder = new Schema.Builder(); for (int fieldId : fieldAccessDescriptor.fieldIdsAccessed()) { builder.addField(inputSchema.getField(fieldId)); } for (Map.Entry<Integer, FieldAccessDescriptor> nested : fieldAccessDescriptor.nestedFields().entrySet()) { Field field = inputSchema.getField(nested.getKey()); FieldAccessDescriptor nestedDescriptor = nested.getValue(); FieldType nestedType = FieldType.row(getOutputSchema(field.getType().getRowSchema(), nestedDescriptor)); if (field.getNullable()) { builder.addNullableField(field.getName(), nestedType); } else { builder.addField(field.getName(), nestedType); } } return builder.build(); }
private static RelDataType toRelDataType( RelDataTypeFactory dataTypeFactory, FieldType fieldType) { switch (fieldType.getTypeName()) { case ARRAY: return dataTypeFactory.createArrayType( toRelDataType(dataTypeFactory, fieldType.getCollectionElementType()), UNLIMITED_ARRAY_SIZE); case MAP: RelDataType componentKeyType = toRelDataType(dataTypeFactory, fieldType.getMapKeyType()); RelDataType componentValueType = toRelDataType(dataTypeFactory, fieldType.getMapValueType()); return dataTypeFactory.createMapType(componentKeyType, componentValueType); case ROW: return toCalciteRowType(fieldType.getRowSchema(), dataTypeFactory); default: return dataTypeFactory.createSqlType(toSqlTypeName(fieldType)); } }
/** Returns the coder used for a given primitive type. */ public static <T> Coder<T> coderForFieldType(FieldType fieldType) { switch (fieldType.getTypeName()) { case ROW: return (Coder<T>) RowCoder.of(fieldType.getRowSchema()); case ARRAY: return (Coder<T>) ListCoder.of(coderForFieldType(fieldType.getCollectionElementType())); case MAP: return (Coder<T>) MapCoder.of( coderForFieldType(fieldType.getMapKeyType()), coderForFieldType(fieldType.getMapValueType())); default: return (Coder<T>) CODER_MAP.get(fieldType.getTypeName()); } }
Schema rowSchema() { return type().getRowSchema(); }
private static Schema getFieldSchema(Field field) { FieldType type = field.getType(); if (TypeName.ROW.equals(type.getTypeName())) { return type.getRowSchema(); } else if (TypeName.ARRAY.equals(type.getTypeName()) && TypeName.ROW.equals(type.getCollectionElementType().getTypeName())) { return type.getCollectionElementType().getRowSchema(); } else if (TypeName.MAP.equals(type.getTypeName()) && TypeName.ROW.equals(type.getMapValueType().getTypeName())) { return type.getMapValueType().getRowSchema(); } else { throw new IllegalArgumentException( "Field " + field + " must be either a row or " + " a container containing rows"); } }
private Schema payloadSchema() { return messageSchema().getField(PAYLOAD_FIELD).getType().getRowSchema(); }
private static Object convertRecordStrict(GenericRecord record, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.ROW, "record"); return toRowStrict(record, fieldType.getRowSchema()); }
@Test public void testNestedSchema() { Schema nestedSchema = Schema.of(Field.of("f1_str", FieldType.STRING)); Schema schema = Schema.of(Field.of("nested", FieldType.row(nestedSchema))); Field inner = schema.getField("nested").getType().getRowSchema().getField("f1_str"); assertEquals("f1_str", inner.getName()); assertEquals(FieldType.STRING, inner.getType()); }
private static StackManipulation getCoder(Schema.FieldType fieldType) { if (TypeName.ARRAY.equals(fieldType.getTypeName())) { return listCoder(fieldType.getCollectionElementType()); } else if (TypeName.MAP.equals(fieldType.getTypeName())) { return mapCoder(fieldType.getMapKeyType(), fieldType.getMapValueType()); } else if (TypeName.ROW.equals(fieldType.getTypeName())) { Coder<Row> nestedCoder = generate(fieldType.getRowSchema(), UUID.randomUUID()); return rowCoder(nestedCoder.getClass()); } else { return coderForPrimitiveType(fieldType.getTypeName()); } }