private static Object toBeamValue(FieldType fieldType, Object jsonBQValue) { if (jsonBQValue instanceof String && JSON_VALUE_PARSERS.containsKey(fieldType.getTypeName())) { return JSON_VALUE_PARSERS.get(fieldType.getTypeName()).apply((String) jsonBQValue); } if (jsonBQValue instanceof List) { return ((List<Object>) jsonBQValue) .stream() .map(v -> ((Map<String, Object>) v).get("v")) .map(v -> toBeamValue(fieldType.getCollectionElementType(), v)) .collect(toList()); } throw new UnsupportedOperationException( "Converting BigQuery type '" + jsonBQValue.getClass() + "' to '" + fieldType + "' is not supported"); } }
private static Object convertArrayStrict( List<Object> values, org.apache.avro.Schema elemAvroSchema, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.ARRAY, "array"); List<Object> ret = new ArrayList<>(values.size()); Schema.FieldType elemFieldType = fieldType.getCollectionElementType(); for (Object value : values) { ret.add(convertAvroFieldStrict(value, elemAvroSchema, elemFieldType)); } return ret; }
@Override public boolean equals(Object o) { if (!(o instanceof FieldType)) { return false; } FieldType other = (FieldType) o; return Objects.equals(getTypeName(), other.getTypeName()) && Objects.equals(getCollectionElementType(), other.getCollectionElementType()) && Objects.equals(getMapKeyType(), other.getMapKeyType()) && Objects.equals(getMapValueType(), other.getMapValueType()) && Objects.equals(getRowSchema(), other.getRowSchema()) && Arrays.equals(getMetadata(), other.getMetadata()); }
private Object verify(Object value, FieldType type, String fieldName) { if (TypeName.ARRAY.equals(type.getTypeName())) { List<Object> arrayElements = verifyArray( value, type.getCollectionElementType(), type.getCollectionElementTypeNullable(), fieldName); return arrayElements; } else if (TypeName.MAP.equals(type.getTypeName())) { Map<Object, Object> mapElements = verifyMap( value, type.getMapKeyType().getTypeName(), type.getMapValueType(), type.getMapValueTypeNullable(), fieldName); return mapElements; } else if (TypeName.ROW.equals(type.getTypeName())) { return verifyRow(value, fieldName); } else { return verifyPrimitiveType(value, type.getTypeName(), fieldName); } }
static <T> T visit(SchemaZipFold<T> zipFold, Context context, FieldType left, FieldType right) { if (left.getTypeName() != right.getTypeName()) { return zipFold.accept(context, left, right); } Context newContext = context.withParent(left.getTypeName()); switch (left.getTypeName()) { case ARRAY: return zipFold.accumulate( zipFold.accept(context, left, right), visit( zipFold, newContext, left.getCollectionElementType(), right.getCollectionElementType())); case ROW: return visitRow(zipFold, newContext, left.getRowSchema(), right.getRowSchema()); case MAP: return zipFold.accumulate( zipFold.accept(context, left, right), visit( zipFold, newContext, left.getCollectionElementType(), right.getCollectionElementType())); default: return zipFold.accept(context, left, right); } }
/** Returns true if two FieldTypes are equal. */ public boolean typesEqual(FieldType other) { if (!Objects.equals(getTypeName(), other.getTypeName())) { return false; } if (!Arrays.equals(getMetadata(), other.getMetadata())) { return false; } if (getTypeName() == TypeName.ARRAY && !getCollectionElementType().typesEqual(other.getCollectionElementType())) { return false; } if (getTypeName() == TypeName.MAP && (!getMapValueType().typesEqual(other.getMapValueType()) || !getMapKeyType().typesEqual(other.getMapKeyType()))) { return false; } if (getTypeName() == TypeName.ROW && !getRowSchema().typesEqual(other.getRowSchema())) { return false; } return true; }
private boolean equivalent(FieldType other) { if (!other.getTypeName().equals(getTypeName())) { return false; } switch (getTypeName()) { case ROW: if (!other.getRowSchema().equivalent(getRowSchema())) { return false; } break; case ARRAY: if (!other.getCollectionElementType().equivalent(getCollectionElementType())) { return false; } break; case MAP: if (!other.getMapKeyType().equivalent(getMapKeyType()) || !other.getMapValueType().equivalent(getMapValueType())) { return false; } break; default: return other.equals(this); } return true; }
static void verifyFieldTypeSupported(Schema.FieldType fieldType) { Schema.TypeName fieldTypeName = fieldType.getTypeName(); if (fieldTypeName.isCompositeType()) { Schema rowFieldSchema = fieldType.getRowSchema(); rowFieldSchema.getFields().forEach(RowJsonValidation::verifyFieldTypeSupported); return; } if (fieldTypeName.isCollectionType()) { verifyFieldTypeSupported(fieldType.getCollectionElementType()); return; } if (!SUPPORTED_TYPES.contains(fieldTypeName)) { throw new RowJsonDeserializer.UnsupportedRowJsonException( fieldTypeName.name() + " is not supported when converting JSON objects to Rows. " + "Supported types are: " + SUPPORTED_TYPES.toString()); } } }
@SuppressWarnings({"TypeParameterUnusedInFormals", "unchecked"}) private <T> T getValue(FieldType type, Object fieldValue, @Nullable Integer cacheKey) { if (type.getTypeName().equals(TypeName.ROW)) { return (T) new RowWithGetters(type.getRowSchema(), fieldValueGetterFactory, fieldValue); } else if (type.getTypeName().equals(TypeName.ARRAY)) { return cacheKey != null ? (T) cachedLists.computeIfAbsent( cacheKey, i -> getListValue(type.getCollectionElementType(), fieldValue)) : (T) getListValue(type.getCollectionElementType(), fieldValue); } else if (type.getTypeName().equals(TypeName.MAP)) { Map map = (Map) fieldValue; return cacheKey != null ? (T) cachedMaps.computeIfAbsent( cacheKey, i -> getMapValue(type.getMapKeyType(), type.getMapValueType(), map)) : (T) getMapValue(type.getMapKeyType(), type.getMapValueType(), map); } else { return (T) fieldValue; } }
private static RelDataType toRelDataType( RelDataTypeFactory dataTypeFactory, FieldType fieldType) { switch (fieldType.getTypeName()) { case ARRAY: return dataTypeFactory.createArrayType( toRelDataType(dataTypeFactory, fieldType.getCollectionElementType()), UNLIMITED_ARRAY_SIZE); case MAP: RelDataType componentKeyType = toRelDataType(dataTypeFactory, fieldType.getMapKeyType()); RelDataType componentValueType = toRelDataType(dataTypeFactory, fieldType.getMapValueType()); return dataTypeFactory.createMapType(componentKeyType, componentValueType); case ROW: return toCalciteRowType(fieldType.getRowSchema(), dataTypeFactory); default: return dataTypeFactory.createSqlType(toSqlTypeName(fieldType)); } }
private static Schema getFieldSchema(Field field) { FieldType type = field.getType(); if (TypeName.ROW.equals(type.getTypeName())) { return type.getRowSchema(); } else if (TypeName.ARRAY.equals(type.getTypeName()) && TypeName.ROW.equals(type.getCollectionElementType().getTypeName())) { return type.getCollectionElementType().getRowSchema(); } else if (TypeName.MAP.equals(type.getTypeName()) && TypeName.ROW.equals(type.getMapValueType().getTypeName())) { return type.getMapValueType().getRowSchema(); } else { throw new IllegalArgumentException( "Field " + field + " must be either a row or " + " a container containing rows"); } }
private String unparseArray(FieldType fieldType) { return "ARRAY<" + unparse(fieldType.getCollectionElementType()) + ">"; }
FieldType arrayElementType() { return type().getCollectionElementType(); }
/** Returns the coder used for a given primitive type. */ public static <T> Coder<T> coderForFieldType(FieldType fieldType) { switch (fieldType.getTypeName()) { case ROW: return (Coder<T>) RowCoder.of(fieldType.getRowSchema()); case ARRAY: return (Coder<T>) ListCoder.of(coderForFieldType(fieldType.getCollectionElementType())); case MAP: return (Coder<T>) MapCoder.of( coderForFieldType(fieldType.getMapKeyType()), coderForFieldType(fieldType.getMapValueType())); default: return (Coder<T>) CODER_MAP.get(fieldType.getTypeName()); } }
/** Get a {@link TypeDescriptor} from a {@link FieldType}. */ public static TypeDescriptor javaTypeForFieldType(FieldType fieldType) { switch (fieldType.getTypeName()) { case ARRAY: return TypeDescriptors.lists(javaTypeForFieldType(fieldType.getCollectionElementType())); case MAP: return TypeDescriptors.maps( javaTypeForFieldType(fieldType.getMapKeyType()), javaTypeForFieldType(fieldType.getMapValueType())); case ROW: return TypeDescriptors.rows(); default: return PRIMITIVE_MAPPING.get(fieldType.getTypeName()); } } /** Get a {@link FieldType} from a {@link TypeDescriptor}. */
static boolean deepEquals(Object a, Object b, Schema.FieldType fieldType) { if (fieldType.getTypeName() == Schema.TypeName.BYTES) { return Arrays.equals((byte[]) a, (byte[]) b); } else if (fieldType.getTypeName() == Schema.TypeName.ARRAY) { return deepEqualsForList( (List<Object>) a, (List<Object>) b, fieldType.getCollectionElementType()); } else if (fieldType.getTypeName() == Schema.TypeName.MAP) { return deepEqualsForMap( (Map<Object, Object>) a, (Map<Object, Object>) b, fieldType.getMapValueType()); } else { return Objects.equals(a, b); } }
private static Object convertAvroArray(Field beamField, Object value) { // Check whether the type of array element is equal. List<Object> values = (List<Object>) value; List<Object> ret = new ArrayList(); for (Object v : values) { ret.add( convertAvroPrimitiveTypes( beamField.getType().getCollectionElementType().getTypeName(), v)); } return (Object) ret; }
static int deepHashCode(Object a, Schema.FieldType fieldType) { if (fieldType.getTypeName() == Schema.TypeName.BYTES) { return Arrays.hashCode((byte[]) a); } else if (fieldType.getTypeName() == Schema.TypeName.ARRAY) { return deepHashCodeForList((List<Object>) a, fieldType.getCollectionElementType()); } else if (fieldType.getTypeName() == Schema.TypeName.MAP) { return deepHashCodeForMap( (Map<Object, Object>) a, fieldType.getMapKeyType(), fieldType.getMapValueType()); } else { return Objects.hashCode(a); } }
@Override public int hashCode() { return Arrays.deepHashCode( new Object[] { getTypeName(), getCollectionElementType(), getMapKeyType(), getMapValueType(), getRowSchema(), getMetadata() }); } }
private static StackManipulation getCoder(Schema.FieldType fieldType) { if (TypeName.ARRAY.equals(fieldType.getTypeName())) { return listCoder(fieldType.getCollectionElementType()); } else if (TypeName.MAP.equals(fieldType.getTypeName())) { return mapCoder(fieldType.getMapKeyType(), fieldType.getMapValueType()); } else if (TypeName.ROW.equals(fieldType.getTypeName())) { Coder<Row> nestedCoder = generate(fieldType.getRowSchema(), UUID.randomUUID()); return rowCoder(nestedCoder.getClass()); } else { return coderForPrimitiveType(fieldType.getTypeName()); } }