public static Field mapToField(Map fMap) { String typeStr = fMap.get("type").toString(); String nameStr = fMap.get("name").toString(); String modeStr = fMap.get("mode").toString(); LegacySQLTypeName type = null; if (typeStr.equals("BOOLEAN")) { type = LegacySQLTypeName.BOOLEAN; } else if (typeStr.equals("STRING")) { type = LegacySQLTypeName.STRING; } else if (typeStr.equals("BYTES")) { type = LegacySQLTypeName.BYTES; } else if (typeStr.equals("INTEGER")) { type = LegacySQLTypeName.INTEGER; } else if (typeStr.equals("FLOAT")) { type = LegacySQLTypeName.FLOAT; } else if (typeStr.equals("TIMESTAMP") || typeStr.equals("DATE") || typeStr.equals("TIME") || typeStr.equals("DATETIME")) { type = LegacySQLTypeName.TIMESTAMP; } else if (typeStr.equals("RECORD")) { type = LegacySQLTypeName.RECORD; } return Field.newBuilder(nameStr, type).setMode(Field.Mode.valueOf(modeStr)).build(); }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); assertEquals(expectedSchema, remoteTable.getDefinition().getSchema()); QueryJobConfiguration config =
@Test public void testIntegerArray() { final String fieldName = "IntegerArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.INTEGER ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.INT32_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
private Field getKafkaDataField() { Field topicField = Field.of("topic", LegacySQLTypeName.STRING); Field partitionField = Field.of("partition", LegacySQLTypeName.INTEGER); Field offsetField = Field.of("offset", LegacySQLTypeName.INTEGER); Field insertTimeField = Field.newBuilder("insertTime",LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(); return Field.newBuilder("kafkaData", LegacySQLTypeName.RECORD, topicField, partitionField, offsetField, insertTimeField) .setMode(Field.Mode.NULLABLE) .build(); } }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
@Test public void testAllFieldsNullable() { final String fieldName = "RequiredField"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.NULLABLE ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, SchemaBuilder.string().required().build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(true).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); } }
private static Field field(TableFieldSchema field) { return Field.newBuilder(field.getName(), fieldType(field)).setMode(mode(field)).build(); }
private com.google.cloud.bigquery.Field.Builder convertMap(Schema kafkaConnectSchema, String fieldName) { Schema keySchema = kafkaConnectSchema.keySchema(); Schema valueSchema = kafkaConnectSchema.valueSchema(); com.google.cloud.bigquery.Field keyField = convertField(keySchema, MAP_KEY_FIELD_NAME).build(); com.google.cloud.bigquery.Field valueField = convertField(valueSchema, MAP_VALUE_FIELD_NAME).build(); com.google.cloud.bigquery.Field.Builder bigQueryRecordBuilder = com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.RECORD, keyField, valueField); return bigQueryRecordBuilder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); }
private com.google.cloud.bigquery.Field.Builder convertStruct(Schema kafkaConnectSchema, String fieldName) { List<com.google.cloud.bigquery.Field> bigQueryRecordFields = new ArrayList<>(); for (Field kafkaConnectField : kafkaConnectSchema.fields()) { com.google.cloud.bigquery.Field.Builder bigQueryRecordFieldBuilder = convertField(kafkaConnectField.schema(), kafkaConnectField.name()); bigQueryRecordFields.add(bigQueryRecordFieldBuilder.build()); } FieldList fieldList = FieldList.of(bigQueryRecordFields); return com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.RECORD, fieldList); }
@Test public void test() { Schema kafkaConnectTestSchema = SchemaBuilder.struct().field("base", Schema.STRING_SCHEMA).build(); Field kafkaDataField = getKafkaDataField(); Field baseField = Field.newBuilder("base", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(); com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of(baseField, kafkaDataField); com.google.cloud.bigquery.Schema bigQueryActualSchema = new KafkaDataBQSchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryActualSchema); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, Field... subFields) { return newBuilder(name, type, subFields).build(); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, FieldList subFields) { return newBuilder(name, type, subFields).build(); }
/** * Returns a Field object with given name and value. */ public static Field of(String name, Type type) { return builder(name, type).build(); }
private Field createField(String name, Enum valueType, String description) { Field.Builder fieldBuilder = Field.newBuilder(name, FEAST_TO_BIGQUERY_TYPE_MAP.get(valueType)); if (description != null) { fieldBuilder.setDescription(description); } return fieldBuilder.build(); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, Field... subFields) { return newBuilder(name, type, subFields).build(); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, FieldList subFields) { return newBuilder(name, type, subFields).build(); }
@Test public void testToAndFromPb() { compareFieldSchemas(FIELD_SCHEMA1, Field.fromPb(FIELD_SCHEMA1.toPb())); compareFieldSchemas(FIELD_SCHEMA2, Field.fromPb(FIELD_SCHEMA2.toPb())); compareFieldSchemas(FIELD_SCHEMA3, Field.fromPb(FIELD_SCHEMA3.toPb())); Field field = Field.newBuilder(FIELD_NAME1, FIELD_TYPE1).build(); compareFieldSchemas(field, Field.fromPb(field.toPb())); }
@Test public void testToBuilder() { compareFieldSchemas(FIELD_SCHEMA1, FIELD_SCHEMA1.toBuilder().build()); compareFieldSchemas(FIELD_SCHEMA2, FIELD_SCHEMA2.toBuilder().build()); compareFieldSchemas(FIELD_SCHEMA3, FIELD_SCHEMA3.toBuilder().build()); Field field = FIELD_SCHEMA1.toBuilder().setDescription("New Description").build(); assertEquals("New Description", field.getDescription()); field = field.toBuilder().setDescription(FIELD_DESCRIPTION1).build(); compareFieldSchemas(FIELD_SCHEMA1, field); }