TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
TableSchema schema = new TableSchema(); List<TableFieldSchema> tableFieldSchema = new ArrayList<TableFieldSchema>(); TableFieldSchema schemaEntry = new TableFieldSchema(); schemaEntry.setName(myFirstFieldName); schemaEntry.setType("STRING"); tableFieldSchema.add(schemaEntry); schema.setFields(tableFieldSchema); Table table = new Table(); table.setSchema(schema); TableReference tableRef = new TableReference(); tableRef.setDatasetId(DATASET_ID); tableRef.setProjectId(PROJECT_ID); tableRef.setTableId(tableId); table.setTableReference(tableRef); try { bigquery.tables().insert(PROJECT_ID, DATASET_ID, table).execute(); } catch (IOException e) { }
if (Objects.equals(fieldSchema.getMode(), "REPEATED")) { TableFieldSchema elementSchema = fieldSchema.clone().setMode("REQUIRED"); @SuppressWarnings("unchecked") List<Map<String, Object>> rawCells = (List<Map<String, Object>>) v; if (fieldSchema.getType().equals("RECORD")) { @SuppressWarnings("unchecked") Map<String, Object> typedV = (Map<String, Object>) v; return getTypedTableRow(fieldSchema.getFields(), typedV); if (fieldSchema.getType().equals("FLOAT")) { return Double.parseDouble((String) v); if (fieldSchema.getType().equals("BOOLEAN")) { return Boolean.parseBoolean((String) v); if (fieldSchema.getType().equals("TIMESTAMP")) { return (String) v;
private TableFieldSchema tryArrayFieldSchema(org.apache.avro.Schema.Field field) { String fieldName = field.name(); TableFieldSchema tableFieldSchema = new TableFieldSchema().setName(fieldName); boolean nullable = AvroUtils.isNullable(field.schema()); if (!nullable) { tableFieldSchema = tableFieldSchema.setMode(REQUIRED_MODE); } org.apache.avro.Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (fieldSchema.getType() == org.apache.avro.Schema.Type.ARRAY) { return tryFieldSchema(tableFieldSchema.setMode(REPEATED_MODE), fieldSchema.getElementType()); } return tryFieldSchema(tableFieldSchema, fieldSchema); }
/** * Validates that a BigQuery column in Avro format can be used as a valid Datastore Entity key * name. */ public static void validateKeyColumn(TableFieldSchema column, Object columnValue) throws IllegalArgumentException { // Entity key name must be different than null if (columnValue == null) { throw new IllegalArgumentException( String.format( "Column [%s] with NULL value cannot be set as Entity name.", column.getName())); } // Entity key names cannot exceed 1500 bytes, the maximum size of an idex-able Datastore // string property: https://cloud.google.com/datastore/docs/concepts/limits if (column.getType().equals("STRING") && columnValue.toString().getBytes().length > MAX_STRING_SIZE_BYTES) { throw new IllegalArgumentException( String.format( "Column [%s] exceeding %d bytes cannot be set as Entity name.", column.getName(), MAX_STRING_SIZE_BYTES)); } // BigQuery column type must be among the supported ones (ex: cannot be RECORD) if (!SUPPORTED_KEY_NAME_TYPES.contains(column.getType())) { throw new IllegalArgumentException( String.format( "Column [%s] of type %s cannot be set as Entity name.", column.getName(), column.getType())); } }
/** @see TableFieldSchema#getName() */ public String getName() { return fieldSchema.getName(); }
private TableFieldSchema tryFieldSchema(TableFieldSchema fieldSchema, org.apache.avro.Schema avroSchema) { fieldSchema = fieldSchema.setType(getBQFieldType(avroSchema)); if (avroSchema.getType() == org.apache.avro.Schema.Type.RECORD) { List<TableFieldSchema> childFields = new ArrayList<>(); List<org.apache.avro.Schema.Field> avroChildFields = avroSchema.getFields(); for (org.apache.avro.Schema.Field avroChildField : avroChildFields) { childFields.add(tryArrayFieldSchema(avroChildField)); } fieldSchema.setFields(childFields); } return fieldSchema; }
checkNotNull(v, "REQUIRED field %s should not be null", fieldSchema.getName()); String bqType = fieldSchema.getType(); Type expectedAvroType = BIG_QUERY_TO_AVRO_TYPES.get(bqType); verifyNotNull(expectedAvroType, "Unsupported BigQuery type: %s", bqType); avroType, bqType, fieldSchema.getName()); switch (fieldSchema.getType()) { case "STRING": case "DATE": case "RECORD": verify(v instanceof GenericRecord, "Expected GenericRecord, got %s", v.getClass()); return convertGenericRecordToTableRow((GenericRecord) v, fieldSchema.getFields()); case "BYTES": verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass()); String.format( "Unexpected BigQuery field schema type %s for field named %s", fieldSchema.getType(), fieldSchema.getName()));
/** @see TableFieldSchema#setMode(String) */ public BigQueryTableFieldSchema setMode(String mode) { fieldSchema.setMode(mode); return this; }
/** @see TableFieldSchema#setType(String) */ public BigQueryTableFieldSchema setType(String type) { fieldSchema.setType(type); return this; }
/** @see TableFieldSchema#setName(String) */ public BigQueryTableFieldSchema setName(String name) { fieldSchema.setName(name); return this; }
public BigQueryTableFieldSchema() { this.fieldSchema = new TableFieldSchema(); }
/** @see TableFieldSchema#getMode() */ public String getMode() { return fieldSchema.getMode(); }
/** @see TableFieldSchema#getType() */ public String getType() { return fieldSchema.getType(); }
private static List<Field> fieldList(TableFieldSchema record) { List<Field> fieldList = new ArrayList<>(); for (TableFieldSchema field : record.getFields()) { fieldList.add(field(field)); } return fieldList; }
/** Helper method to build the table schema for the output table. */ private static TableSchema buildWeatherSchemaProjection() { List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("year").setType("INTEGER")); fields.add(new TableFieldSchema().setName("month").setType("INTEGER")); fields.add(new TableFieldSchema().setName("day").setType("INTEGER")); fields.add(new TableFieldSchema().setName("mean_temp").setType("FLOAT")); TableSchema schema = new TableSchema().setFields(fields); return schema; }
if (Objects.equals(fieldSchema.getMode(), "REPEATED")) { TableFieldSchema elementSchema = fieldSchema.clone().setMode("REQUIRED"); @SuppressWarnings("unchecked") List<Map<String, Object>> rawCells = (List<Map<String, Object>>) v; if (fieldSchema.getType().equals("RECORD")) { @SuppressWarnings("unchecked") Map<String, Object> typedV = (Map<String, Object>) v; return getTypedTableRow(fieldSchema.getFields(), typedV); if (fieldSchema.getType().equals("FLOAT")) { return Double.parseDouble((String) v); if (fieldSchema.getType().equals("BOOLEAN")) { return Boolean.parseBoolean((String) v); if (fieldSchema.getType().equals("TIMESTAMP")) { return BigQueryAvroUtils.formatTimestamp((String) v);
private TableFieldSchema tryArrayFieldSchema(org.apache.avro.Schema.Field field) { String fieldName = field.name(); TableFieldSchema tableFieldSchema = new TableFieldSchema().setName(fieldName); boolean nullable = AvroUtils.isNullable(field.schema()); if (!nullable) { tableFieldSchema = tableFieldSchema.setMode(REQUIRED_MODE); } org.apache.avro.Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (fieldSchema.getType() == org.apache.avro.Schema.Type.ARRAY) { return tryFieldSchema(tableFieldSchema.setMode(REPEATED_MODE), fieldSchema.getElementType()); } return tryFieldSchema(tableFieldSchema, fieldSchema); }
String columnName = column.getName(); if (columnValue == null) { return Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); } else { Value.Builder valueBuilder = Value.newBuilder(); switch (column.getType()) { case "STRING": "Column [%s] of type [%s] not supported.", column.getName(), column.getType()));