private static Object convertBytesStrict(ByteBuffer bb, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.BYTES, "bytes"); byte[] bytes = new byte[bb.remaining()]; bb.get(bytes); return bytes; }
private static Object convertArrayStrict( List<Object> values, org.apache.avro.Schema elemAvroSchema, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.ARRAY, "array"); List<Object> ret = new ArrayList<>(values.size()); Schema.FieldType elemFieldType = fieldType.getCollectionElementType(); for (Object value : values) { ret.add(convertAvroFieldStrict(value, elemAvroSchema, elemFieldType)); } return ret; }
private static Object convertStringStrict(CharSequence value, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.STRING, "string"); return value.toString(); }
static boolean deepEquals(Object a, Object b, Schema.FieldType fieldType) { if (fieldType.getTypeName() == Schema.TypeName.BYTES) { return Arrays.equals((byte[]) a, (byte[]) b); } else if (fieldType.getTypeName() == Schema.TypeName.ARRAY) { return deepEqualsForList( (List<Object>) a, (List<Object>) b, fieldType.getCollectionElementType()); } else if (fieldType.getTypeName() == Schema.TypeName.MAP) { return deepEqualsForMap( (Map<Object, Object>) a, (Map<Object, Object>) b, fieldType.getMapValueType()); } else { return Objects.equals(a, b); } }
TypeName typeName() { return type().getTypeName(); }
private static Object convertBooleanStrict(Boolean value, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.BOOLEAN, "boolean"); return value; }
private static Object convertFloatStrict(Float value, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.FLOAT, "float"); return value; }
private static Object convertIntStrict(Integer value, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.INT32, "int"); return value; }
@Override public int hashCode() { return Arrays.deepHashCode( new Object[] { getTypeName(), getCollectionElementType(), getMapKeyType(), getMapValueType(), getRowSchema(), getMetadata() }); } }
private static Object convertAvroArray(Field beamField, Object value) { // Check whether the type of array element is equal. List<Object> values = (List<Object>) value; List<Object> ret = new ArrayList(); for (Object v : values) { ret.add( convertAvroPrimitiveTypes( beamField.getType().getCollectionElementType().getTypeName(), v)); } return (Object) ret; }
static int deepHashCode(Object a, Schema.FieldType fieldType) { if (fieldType.getTypeName() == Schema.TypeName.BYTES) { return Arrays.hashCode((byte[]) a); } else if (fieldType.getTypeName() == Schema.TypeName.ARRAY) { return deepHashCodeForList((List<Object>) a, fieldType.getCollectionElementType()); } else if (fieldType.getTypeName() == Schema.TypeName.MAP) { return deepHashCodeForMap( (Map<Object, Object>) a, fieldType.getMapKeyType(), fieldType.getMapValueType()); } else { return Objects.hashCode(a); } }
private void testUnsupportedConversion(FieldType fieldType, String jsonFieldValue) throws Exception { String fieldName = "f_" + fieldType.getTypeName().name().toLowerCase(); ObjectMapper jsonParser = newObjectMapperWith(RowJsonDeserializer.forSchema(schemaWithField(fieldName, fieldType))); thrown.expectMessage(fieldName); thrown.expectCause(unsupportedWithMessage(jsonFieldValue, "out of range")); jsonParser.readValue(jsonObjectWith(fieldName, jsonFieldValue), Row.class); }
private String unparse(FieldType fieldType) { if (fieldType.getTypeName().isMapType()) { return unparseMap(fieldType); } else if (fieldType.getTypeName().isCollectionType()) { return unparseArray(fieldType); } else if (fieldType.getTypeName().isCompositeType()) { return unparseRow(fieldType); } else { return unparsePrimitive(fieldType); } }
@Override public boolean equals(Object o) { if (!(o instanceof FieldType)) { return false; } FieldType other = (FieldType) o; return Objects.equals(getTypeName(), other.getTypeName()) && Objects.equals(getCollectionElementType(), other.getCollectionElementType()) && Objects.equals(getMapKeyType(), other.getMapKeyType()) && Objects.equals(getMapValueType(), other.getMapValueType()) && Objects.equals(getRowSchema(), other.getRowSchema()) && Arrays.equals(getMetadata(), other.getMetadata()); }
private void testSupportedConversion( FieldType fieldType, String jsonFieldValue, Object expectedRowFieldValue) throws Exception { String fieldName = "f_" + fieldType.getTypeName().name().toLowerCase(); Schema schema = schemaWithField(fieldName, fieldType); Row expectedRow = Row.withSchema(schema).addValues(expectedRowFieldValue).build(); ObjectMapper jsonParser = newObjectMapperWith(RowJsonDeserializer.forSchema(schema)); Row parsedRow = jsonParser.readValue(jsonObjectWith(fieldName, jsonFieldValue), Row.class); assertEquals(expectedRow, parsedRow); }
private static Object convertLongStrict(Long value, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.INT64, "long"); return value; }
private static Object convertRecordStrict(GenericRecord record, Schema.FieldType fieldType) { checkTypeName(fieldType.getTypeName(), Schema.TypeName.ROW, "record"); return toRowStrict(record, fieldType.getRowSchema()); }
static StackManipulation mapCoder(Schema.FieldType keyType, Schema.FieldType valueType) { StackManipulation keyCoder = coderForPrimitiveType(keyType.getTypeName()); StackManipulation valueCoder = getCoder(valueType); return new Compound( keyCoder, valueCoder, MethodInvocation.invoke( MAP_CODER_TYPE.getDeclaredMethods().filter(ElementMatchers.named("of")).getOnly())); }
private static StackManipulation getCoder(Schema.FieldType fieldType) { if (TypeName.ARRAY.equals(fieldType.getTypeName())) { return listCoder(fieldType.getCollectionElementType()); } else if (TypeName.MAP.equals(fieldType.getTypeName())) { return mapCoder(fieldType.getMapKeyType(), fieldType.getMapValueType()); } else if (TypeName.ROW.equals(fieldType.getTypeName())) { Coder<Row> nestedCoder = generate(fieldType.getRowSchema(), UUID.randomUUID()); return rowCoder(nestedCoder.getClass()); } else { return coderForPrimitiveType(fieldType.getTypeName()); } }
private static void unnestRow(Row input, Row.Builder output) { for (int i = 0; i < input.getSchema().getFieldCount(); ++i) { Field field = input.getSchema().getField(i); if (field.getType().getTypeName().isCompositeType()) { unnestRow(input.getRow(i), output); } else { output.addValue(input.getValue(i)); } } } /** A {@link PTransform} that unnests nested row. */