TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
private TableFieldSchema tryFieldSchema(TableFieldSchema fieldSchema, org.apache.avro.Schema avroSchema) { fieldSchema = fieldSchema.setType(getBQFieldType(avroSchema)); if (avroSchema.getType() == org.apache.avro.Schema.Type.RECORD) { List<TableFieldSchema> childFields = new ArrayList<>(); List<org.apache.avro.Schema.Field> avroChildFields = avroSchema.getFields(); for (org.apache.avro.Schema.Field avroChildField : avroChildFields) { childFields.add(tryArrayFieldSchema(avroChildField)); } fieldSchema.setFields(childFields); } return fieldSchema; }
private TableFieldSchema tryFieldSchema(TableFieldSchema fieldSchema, org.apache.avro.Schema avroSchema) { fieldSchema = fieldSchema.setType(getBQFieldType(avroSchema)); if (avroSchema.getType() == org.apache.avro.Schema.Type.RECORD) { List<TableFieldSchema> childFields = new ArrayList<>(); List<org.apache.avro.Schema.Field> avroChildFields = avroSchema.getFields(); for (org.apache.avro.Schema.Field avroChildField : avroChildFields) { childFields.add(tryArrayFieldSchema(avroChildField)); } fieldSchema.setFields(childFields); } return fieldSchema; }
"Expected non-null entry for key 'fields' in JsonObject of type RECORD: '%s'", jsonObject); fieldDef.setFields(getSchemaFromString(jsonObject.get("fields").toString()));
/** Defines the BigQuery schema used for the output. */ static TableSchema getSchema() { List<TableFieldSchema> tagFields = new ArrayList<>(); tagFields.add(new TableFieldSchema().setName("count").setType("INTEGER")); tagFields.add(new TableFieldSchema().setName("tag").setType("STRING")); List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("prefix").setType("STRING")); fields.add( new TableFieldSchema() .setName("tags") .setType("RECORD") .setMode("REPEATED") .setFields(tagFields)); return new TableSchema().setFields(fields); } }
tfs.setFields(subfields);
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
List<TableFieldSchema> fields = new ArrayList<>(); TableFieldSchema column = new TableFieldSchema().setName("rawFields"); column.setType("RECORD"); List<TableFieldSchema> list = new ArrayList<>(); list.add(new TableFieldSchema().setName("key").setType("STRING")); list.add(new TableFieldSchema().setName("value").setType("STRING")); column.setFields(list); column.setMode("REPEATED"); fields.add(column); TableSchema schema = new TableSchema().setFields(fields); TableRow row = new TableRow(); List<TableRow> rawFields = new ArrayList<>(); rawFields.add(new TableRow().set("key","foo").set("value", "bar")); row.set("rawFields", rawFields); Pipeline p = Pipeline.create(options); PCollection<TableRow> c = p.apply(Create.of(row, row).withCoder(TableRowJsonCoder.of())); c.apply(BigQueryIO.Write.named("BigQuery-Write") .to(options.getOutput()) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withSchema(schema)); p.run();
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (fields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (fields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
static TableFieldSchema getAlignmentFieldSchema(String fieldName) { TableFieldSchema fieldSchema = new TableFieldSchema().setName(fieldName).setType(RECORD); fieldSchema.setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; { // Position record. add(getPositionFieldSchema(POSITION)); add(new TableFieldSchema().setName(MAPPING_QUALITY).setType(INT64)); // CIGAR record. TableFieldSchema cigarSchema = new TableFieldSchema().setName(CIGAR).setType(RECORD).setMode(REPEATED); cigarSchema.setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; { // Operation is of type enum, but BQ doesn't accept enum. Hence, converting to INT64. add(new TableFieldSchema().setName(OPERATION).setType(INT64)); add(new TableFieldSchema().setName(OPERATION_LENGTH).setType(INT64)); add(new TableFieldSchema().setName(REFERENCE_SEQUENCE).setType(STRING)); } }); add(cigarSchema); } }); return fieldSchema; }
private static List<TableFieldSchema> toTableFieldSchema(Schema schema) { List<TableFieldSchema> fields = new ArrayList<>(schema.getFieldCount()); for (Field schemaField : schema.getFields()) { FieldType type = schemaField.getType(); TableFieldSchema field = new TableFieldSchema().setName(schemaField.getName()); if (schemaField.getDescription() != null && !"".equals(schemaField.getDescription())) { field.setDescription(schemaField.getDescription()); } if (!schemaField.getNullable()) { field.setMode(Mode.REQUIRED.toString()); } if (TypeName.ARRAY == type.getTypeName()) { type = type.getCollectionElementType(); field.setMode(Mode.REPEATED.toString()); } if (TypeName.ROW == type.getTypeName()) { Schema subType = type.getRowSchema(); field.setFields(toTableFieldSchema(subType)); } field.setType(toStandardSQLTypeName(type).toString()); fields.add(field); } return fields; }
static TableFieldSchema getPositionFieldSchema(String fieldName) { TableFieldSchema fieldSchema = new TableFieldSchema().setName(fieldName).setType(RECORD); fieldSchema.setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; { add(new TableFieldSchema().setName(REFERENCE_NAME).setType(STRING)); add(new TableFieldSchema().setName(POSITION).setType(INT64)); add(new TableFieldSchema().setName(REVERSE_STRAND).setType(BOOL)); } }); return fieldSchema; }