TableSchema toPb() { TableSchema tableSchemaPb = new TableSchema(); if (fields != null) { tableSchemaPb.setFields(fields.toPb()); } return tableSchemaPb; }
static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { List<TableFieldSchema> fields = tableSchemaPb.getFields(); if (fields == null) { fields = Collections.emptyList(); } return Schema.of(FieldList.fromPb(fields)); } }
TableSchema schema = new TableSchema(); schema.setFields(new ArrayList<TableFieldSchema>()); tfs.setFields(subfields); schema.getFields().add(tfs);
@Test public void testEmptySchema() { TableSchema tableSchema = new TableSchema(); Schema schema = Schema.fromPb(tableSchema); assertEquals(0, schema.getFields().size()); } }
/** @see TableSchema#setFields(List) */ public BigQueryTableSchema setFields(List<BigQueryTableFieldSchema> bigQueryTableFields) { Preconditions.checkArgument(!bigQueryTableFields.isEmpty(), "Empty fields."); List<TableFieldSchema> fields = new ArrayList<>(bigQueryTableFields.size()); for (BigQueryTableFieldSchema bigQueryTableField : bigQueryTableFields) { fields.add(bigQueryTableField.get()); } tableSchema.setFields(fields); return this; }
@Test public void testBuildWriteDisplayData() { String tableSpec = "project:dataset.table"; TableSchema schema = new TableSchema().set("col1", "type1").set("col2", "type2"); final String tblDescription = "foo bar table"; BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows() .to(tableSpec) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) .withTableDescription(tblDescription) .withoutValidation(); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("table")); assertThat(displayData, hasDisplayItem("schema")); assertThat( displayData, hasDisplayItem( "createDisposition", BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED.toString())); assertThat( displayData, hasDisplayItem( "writeDisposition", BigQueryIO.Write.WriteDisposition.WRITE_APPEND.toString())); assertThat(displayData, hasDisplayItem("tableDescription", tblDescription)); assertThat(displayData, hasDisplayItem("validation", false)); }
.setTableId(tableId)); tableService.insert(projectId, datasetId, newTable).execute(); } else if (!table.getSchema().equals(schema)) { throw new RuntimeException( "Table exists and schemas do not match, expecting: " + schema.toPrettyString() + ", actual: " + table.getSchema().toPrettyString());
@Override public boolean equals(Object object) { if (!(object instanceof BigQueryTableSchema)) { return false; } BigQueryTableSchema another = (BigQueryTableSchema) object; return tableSchema.equals(another.tableSchema); }
@Test public void testConvertBigQuerySchemaToAvroSchema() { TableSchema tableSchema = new TableSchema(); tableSchema.setFields(fields); Schema avroSchema = BigQueryAvroUtils.toGenericAvroSchema("testSchema", tableSchema.getFields());
public BigQueryTableSchema() { this.tableSchema = new TableSchema(); }
com.google.api.services.bigquery.model.TableSchema toPb() { com.google.api.services.bigquery.model.TableSchema tableSchemaPb = new com.google.api.services.bigquery.model.TableSchema(); if (fields != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields, Field.TO_PB_FUNCTION); tableSchemaPb.setFields(fieldsPb); } return tableSchemaPb; }
/** * Utility function to convert from an Avro {@link GenericRecord} to a BigQuery {@link TableRow}. * * <p>See <a href="https://cloud.google.com/bigquery/exporting-data-from-bigquery#config">"Avro * format"</a> for more information. */ static TableRow convertGenericRecordToTableRow(GenericRecord record, TableSchema schema) { return convertGenericRecordToTableRow(record, schema.getFields()); }
@Override public TableSchema getSchema(String destination) { return new TableSchema(); } }
com.google.api.services.bigquery.model.TableSchema toPb() { com.google.api.services.bigquery.model.TableSchema tableSchemaPb = new com.google.api.services.bigquery.model.TableSchema(); if (fields != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields, Field.TO_PB_FUNCTION); tableSchemaPb.setFields(fieldsPb); } return tableSchemaPb; }
static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { return Schema.of(Lists.transform(tableSchemaPb.getFields(), Field.FROM_PB_FUNCTION)); } }
.to(tableRef) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(new TableSchema()) .withTestServices(fakeBqServices)); p.run();
TableSchema toPb() { TableSchema tableSchemaPb = new TableSchema(); if (fields != null) { tableSchemaPb.setFields(fields.toPb()); } return tableSchemaPb; }
static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { return Schema.of(Lists.transform(tableSchemaPb.getFields(), Field.FROM_PB_FUNCTION)); } }
@Test public void testWriteBuilderMethods() { BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows().to("foo.com:project:somedataset.sometable"); assertEquals("foo.com:project", write.getTable().get().getProjectId()); assertEquals("somedataset", write.getTable().get().getDatasetId()); assertEquals("sometable", write.getTable().get().getTableId()); assertNull(write.getJsonSchema()); assertNull(write.getSchemaFromView()); assertEquals(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED, write.getCreateDisposition()); assertEquals(BigQueryIO.Write.WriteDisposition.WRITE_EMPTY, write.getWriteDisposition()); assertEquals(null, write.getTableDescription()); assertEquals(true, write.getValidate()); assertFalse(write.withoutValidation().getValidate()); TableSchema schema = new TableSchema(); assertEquals( schema, BigQueryHelpers.fromJsonString( write.withSchema(schema).getJsonSchema().get(), TableSchema.class)); }
/** Convert a Beam {@link Schema} to a BigQuery {@link TableSchema}. */ public static TableSchema toTableSchema(Schema schema) { return new TableSchema().setFields(toTableFieldSchema(schema)); }