public static Object autoCastField(Schema.Field field, Object rawObj) { if (rawObj == null) { if (!field.getNullable()) { throw new IllegalArgumentException(String.format("Field %s not nullable", field.getName())); TypeName type = field.getType().getTypeName(); if (type.isStringType()) { if (rawObj instanceof NlsString) {
public Builder addArrayField(String name, FieldType collectionElementType) { fields.add(Field.of(name, FieldType.array(collectionElementType))); return this; }
@Override public int hashCode() { return Objects.hash(getName(), getDescription(), getType(), getNullable()); } }
assertEquals( Stream.of( Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of( "address", Schema.FieldType.row( .addNullableField("country", VARCHAR) .build())) .withNullable(true), Field.of( "addressAngular", Schema.FieldType.row( .addNullableField("country", VARCHAR) .build())) .withNullable(true), Field.of("isRobot", BOOLEAN).withNullable(true)) .collect(toSchema()), table.getSchema());
assertEquals("f_byte", schema.getField(0).getName()); assertEquals(FieldType.BYTE, schema.getField(0).getType()); assertEquals("f_int16", schema.getField(1).getName()); assertEquals(FieldType.INT16, schema.getField(1).getType()); assertEquals("f_int32", schema.getField(2).getName()); assertEquals(FieldType.INT32, schema.getField(2).getType()); assertEquals("f_int64", schema.getField(3).getName()); assertEquals(FieldType.INT64, schema.getField(3).getType()); assertEquals("f_decimal", schema.getField(4).getName()); assertEquals(FieldType.DECIMAL, schema.getField(4).getType()); assertEquals("f_float", schema.getField(5).getName()); assertEquals(FieldType.FLOAT, schema.getField(5).getType()); assertEquals("f_double", schema.getField(6).getName()); assertEquals(FieldType.DOUBLE, schema.getField(6).getType()); assertEquals("f_string", schema.getField(7).getName()); assertEquals(FieldType.STRING, schema.getField(7).getType()); assertEquals("f_datetime", schema.getField(8).getName()); assertEquals(FieldType.DATETIME, schema.getField(8).getType()); assertEquals("f_boolean", schema.getField(9).getName()); assertEquals(FieldType.BOOLEAN, schema.getField(9).getType());
Schema type = Stream.of( Schema.Field.of("f_byte", FieldType.BYTE).withNullable(true), Schema.Field.of("f_int16", FieldType.INT16).withNullable(true), Schema.Field.of("f_int32", FieldType.INT32).withNullable(true), Schema.Field.of("f_int64", FieldType.INT64).withNullable(true), Schema.Field.of("f_decimal", FieldType.DECIMAL).withNullable(true), Schema.Field.of("f_float", FieldType.FLOAT).withNullable(true), Schema.Field.of("f_double", FieldType.DOUBLE).withNullable(true), Schema.Field.of("f_string", FieldType.STRING).withNullable(true), Schema.Field.of("f_datetime", FieldType.DATETIME).withNullable(true), Schema.Field.of("f_boolean", FieldType.BOOLEAN).withNullable(true), Schema.Field.of("f_array", FieldType.array(FieldType.DATETIME)).withNullable(true), Schema.Field.of("f_map", FieldType.map(FieldType.INT32, FieldType.DOUBLE)) .withNullable(true)) .collect(toSchema());
FieldType type = schema.getField(i).getType(); FieldValueSetter setter = setters.get(i); setter.set(
Lists.newArrayListWithCapacity(schema.getFieldCount()); for (int i = 0; i < schema.getFieldCount(); i++) { componentCoders.add(getCoder(schema.getField(i).getType()));
public static Object convertAvroFormat(Field beamField, Object value) throws RuntimeException { Object ret; TypeName beamFieldTypeName = beamField.getType().getTypeName(); switch (beamFieldTypeName) { case INT16:
@Test public void testExecute_createTableWithPrefixMapField() throws Exception { InMemoryMetaStore metaStore = new InMemoryMetaStore(); metaStore.registerProvider(new TextTableProvider()); BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore); cli.execute( "CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age', \n" + "tags MAP<VARCHAR, VARCHAR>, \n" + "nestedMap MAP<INTEGER, MAP<VARCHAR, INTEGER>> \n" + ") \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'"); Table table = metaStore.getTables().get("person"); assertNotNull(table); assertEquals( Stream.of( Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of("tags", Schema.FieldType.map(VARCHAR, VARCHAR)).withNullable(true), Field.of( "nestedMap", Schema.FieldType.map(INTEGER, Schema.FieldType.map(VARCHAR, INTEGER))) .withNullable(true)) .collect(toSchema()), table.getSchema()); }
static Row selectRow( Row input, FieldAccessDescriptor fieldAccessDescriptor, Schema inputSchema, Schema outputSchema) { if (fieldAccessDescriptor.allFields()) { return input; } else { Row.Builder output = Row.withSchema(outputSchema); for (int fieldId : fieldAccessDescriptor.fieldIdsAccessed()) { output.addValue(input.getValue(fieldId)); } for (Map.Entry<Integer, FieldAccessDescriptor> nested : fieldAccessDescriptor.nestedFields().entrySet()) { String fieldName = inputSchema.nameOf(nested.getKey()); Schema nestedInputSchema = inputSchema.getField(nested.getKey()).getType().getRowSchema(); Schema nestedOutputSchema = outputSchema.getField(fieldName).getType().getRowSchema(); output.addValue( selectRow( input.getValue(fieldName), nested.getValue(), nestedInputSchema, nestedOutputSchema)); } return output.build(); } } }
@Test public void testExecute_createTableWithPrefixArrayField() throws Exception { InMemoryMetaStore metaStore = new InMemoryMetaStore(); metaStore.registerProvider(new TextTableProvider()); BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore); cli.execute( "CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age', \n" + "tags ARRAY<VARCHAR>, \n" + "matrix ARRAY<ARRAY<INTEGER>> \n" + ") \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'"); Table table = metaStore.getTables().get("person"); assertNotNull(table); assertEquals( Stream.of( Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of("tags", Schema.FieldType.array(VARCHAR)).withNullable(true), Field.of("matrix", Schema.FieldType.array(Schema.FieldType.array(INTEGER))) .withNullable(true)) .collect(toSchema()), table.getSchema()); }
/** * Return the list of {@link FieldValueGetter}s for a Java Bean class * * <p>The returned list is ordered by the order of fields in the schema. */ public static List<FieldValueGetter> getGetters(Class<?> clazz, Schema schema) { return CACHED_GETTERS.computeIfAbsent( new ClassWithSchema(clazz, schema), c -> { try { Map<String, FieldValueGetter> getterMap = ReflectUtils.getMethods(clazz) .stream() .filter(ReflectUtils::isGetter) .map(JavaBeanUtils::createGetter) .collect(Collectors.toMap(FieldValueGetter::name, Function.identity())); return schema .getFields() .stream() .map(f -> getterMap.get(f.getName())) .collect(Collectors.toList()); } catch (IOException e) { throw new RuntimeException(e); } }); }
private static List<TableFieldSchema> toTableFieldSchema(Schema schema) { List<TableFieldSchema> fields = new ArrayList<>(schema.getFieldCount()); for (Field schemaField : schema.getFields()) { FieldType type = schemaField.getType(); TableFieldSchema field = new TableFieldSchema().setName(schemaField.getName()); if (schemaField.getDescription() != null && !"".equals(schemaField.getDescription())) { field.setDescription(schemaField.getDescription()); } if (!schemaField.getNullable()) { field.setMode(Mode.REQUIRED.toString()); } if (TypeName.ARRAY == type.getTypeName()) { type = type.getCollectionElementType(); field.setMode(Mode.REPEATED.toString()); } if (TypeName.ROW == type.getTypeName()) { Schema subType = type.getRowSchema(); field.setFields(toTableFieldSchema(subType)); } field.setType(toStandardSQLTypeName(type).toString()); fields.add(field); } return fields; }
/** * Return the list of {@link FieldValueSetter}s for a Java Bean class * * <p>The returned list is ordered by the order of fields in the schema. */ public static List<FieldValueSetter> getSetters(Class<?> clazz, Schema schema) { return CACHED_SETTERS.computeIfAbsent( new ClassWithSchema(clazz, schema), c -> { try { Map<String, FieldValueSetter> setterMap = ReflectUtils.getMethods(clazz) .stream() .filter(ReflectUtils::isSetter) .map(JavaBeanUtils::createSetter) .collect(Collectors.toMap(FieldValueSetter::name, Function.identity())); return schema .getFields() .stream() .map(f -> setterMap.get(f.getName())) .collect(Collectors.toList()); } catch (IOException e) { throw new RuntimeException(e); } }); }
private static Row jsonObjectToRow(FieldValue rowFieldValue) { if (!rowFieldValue.isJsonObject()) { throw new UnsupportedRowJsonException( "Expected JSON object for field '" + rowFieldValue.name() + "'. " + "Unable to convert '" + rowFieldValue.jsonValue().asText() + "'" + " to Beam Row, it is not a JSON object. Currently only JSON objects " + "can be parsed to Beam Rows"); } return rowFieldValue .rowSchema() .getFields() .stream() .map( schemaField -> extractJsonNodeValue( FieldValue.of( schemaField.getName(), schemaField.getType(), rowFieldValue.jsonFieldValue(schemaField.getName())))) .collect(toRow(rowFieldValue.rowSchema())); }
@Test public void testExecute_createTextTable() throws Exception { InMemoryMetaStore metaStore = new InMemoryMetaStore(); metaStore.registerProvider(new TextTableProvider()); BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore); cli.execute( "CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age') \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'"); Table table = metaStore.getTables().get("person"); assertNotNull(table); assertEquals( Stream.of( Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true)) .collect(toSchema()), table.getSchema()); }
private List<Object> verify(Schema schema, List<Object> values) { List<Object> verifiedValues = Lists.newArrayListWithCapacity(values.size()); if (schema.getFieldCount() != values.size()) { throw new IllegalArgumentException( String.format( "Field count in Schema (%s) and values (%s) must match", schema.getFieldNames(), values)); } for (int i = 0; i < values.size(); ++i) { Object value = values.get(i); Schema.Field field = schema.getField(i); if (value == null) { if (!field.getNullable()) { throw new IllegalArgumentException( String.format("Field %s is not nullable", field.getName())); } verifiedValues.add(null); } else { verifiedValues.add(verify(value, field.getType(), field.getName())); } } return verifiedValues; }