/** Example of copying multiple tables to a destination. */ public void copyTables(String datasetId, String destinationTableId) throws InterruptedException { generateTableWithDdl(datasetId, "table1"); generateTableWithDdl(datasetId, "table2"); // [START bigquery_copy_table_multiple_source] TableId destinationTable = TableId.of(datasetId, destinationTableId); CopyJobConfiguration configuration = CopyJobConfiguration.newBuilder( destinationTable, Arrays.asList(TableId.of(datasetId, "table1"), TableId.of(datasetId, "table2"))) .build(); // Copy the tables. Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); // Check the table StandardTableDefinition table = bigquery.getTable(destinationTable).getDefinition(); System.out.println("State: " + job.getStatus().getState()); System.out.printf("Copied %d rows.\n", table.getNumRows()); // [END bigquery_copy_table_multiple_source] }
/** Example of loading a parquet file from GCS to a table. */ public void loadTableGcsParquet(String datasetName) throws InterruptedException { // [START bigquery_load_table_gcs_parquet] String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"; TableId tableId = TableId.of(datasetName, "us_states"); LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, sourceUri) .setFormatOptions(FormatOptions.parquet()) .build(); // Load the table Job loadJob = bigquery.create(JobInfo.of(configuration)); loadJob = loadJob.waitFor(); // Check the table StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition(); System.out.println("State: " + loadJob.getStatus().getState()); System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows()); // [END bigquery_load_table_gcs_parquet] }
bigquery.getTable(TableId.of(datasetId, recoverTableId)).getDefinition(); System.out.println("State: " + job.getStatus().getState()); System.out.printf("Recovered %d rows.\n", table.getNumRows());
@Test public void testCreateAndGetTableWithSelectedField() { String tableName = "test_create_and_get_selected_fields_table"; TableId tableId = TableId.of(DATASET, tableName); StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); Table createdTable = bigquery.create( TableInfo.newBuilder(tableId, tableDefinition) .setLabels(Collections.singletonMap("a", "b")) .build()); assertNotNull(createdTable); assertEquals(DATASET, createdTable.getTableId().getDataset()); assertEquals(tableName, createdTable.getTableId().getTable()); Table remoteTable = bigquery.getTable( DATASET, tableName, TableOption.fields(TableField.CREATION_TIME, TableField.LABELS)); assertNotNull(remoteTable); assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); assertEquals(createdTable.getTableId(), remoteTable.getTableId()); assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); assertThat(remoteTable.getLabels()).containsExactly("a", "b"); assertNotNull(remoteTable.getCreationTime()); assertNull(remoteTable.getDefinition().getSchema()); assertNull(remoteTable.getLastModifiedTime()); assertNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes()); assertNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows()); assertNull(remoteTable.<StandardTableDefinition>getDefinition().getTimePartitioning()); assertNull(remoteTable.<StandardTableDefinition>getDefinition().getClustering()); assertTrue(remoteTable.delete()); }
return ((StandardTableDefinition) bigquery.getTable(tableId).getDefinition()).getNumRows();
Table remoteTable = bigquery.getTable(DATASET, tableName); assertNotNull(remoteTable); assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); assertEquals(createdTable.getTableId(), remoteTable.getTableId()); assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); assertNotNull(remoteTable.getCreationTime()); assertNotNull(remoteTable.getLastModifiedTime()); assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes()); assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows()); assertEquals( partitioning, remoteTable.<StandardTableDefinition>getDefinition().getTimePartitioning()); assertEquals(clustering, remoteTable.<StandardTableDefinition>getDefinition().getClustering()); assertTrue(remoteTable.delete());
@Test public void testUpdateTableWithSelectedFields() { String tableName = "test_update_with_selected_fields_table"; StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition); Table createdTable = bigquery.create(tableInfo); assertNotNull(createdTable); Table updatedTable = bigquery.update( tableInfo.toBuilder().setDescription("newDescr").build(), TableOption.fields(TableField.DESCRIPTION)); assertTrue(updatedTable.getDefinition() instanceof StandardTableDefinition); assertEquals(DATASET, updatedTable.getTableId().getDataset()); assertEquals(tableName, updatedTable.getTableId().getTable()); assertEquals("newDescr", updatedTable.getDescription()); assertNull(updatedTable.getDefinition().getSchema()); assertNull(updatedTable.getLastModifiedTime()); assertNull(updatedTable.<StandardTableDefinition>getDefinition().getNumBytes()); assertNull(updatedTable.<StandardTableDefinition>getDefinition().getNumRows()); assertTrue(createdTable.delete()); }
assertThat(table.getDefinition()).isInstanceOf(StandardTableDefinition.class); assertThat( ((StandardTableDefinition) table.getDefinition()) .getTimePartitioning() .getExpirationMs()) .update(BigQuery.TableOption.fields(BigQuery.TableField.TIME_PARTITIONING)); assertThat( ((StandardTableDefinition) table.getDefinition()) .getTimePartitioning() .getExpirationMs()) .update(BigQuery.TableOption.fields(BigQuery.TableField.TIME_PARTITIONING)); assertThat( ((StandardTableDefinition) table.getDefinition()) .getTimePartitioning() .getExpirationMs())
@Test public void testBuilder() { initializeExpectedTable(2); replay(bigquery); Table builtTable = new Table.Builder(serviceMockReturnsOptions, TABLE_ID1, TABLE_DEFINITION) .setCreationTime(CREATION_TIME) .setDescription(DESCRIPTION) .setEtag(ETAG) .setExpirationTime(EXPIRATION_TIME) .setFriendlyName(FRIENDLY_NAME) .setGeneratedId(GENERATED_ID) .setLastModifiedTime(LAST_MODIFIED_TIME) .setSelfLink(SELF_LINK) .build(); assertEquals(TABLE_ID1, builtTable.getTableId()); assertEquals(CREATION_TIME, builtTable.getCreationTime()); assertEquals(DESCRIPTION, builtTable.getDescription()); assertEquals(ETAG, builtTable.getEtag()); assertEquals(EXPIRATION_TIME, builtTable.getExpirationTime()); assertEquals(FRIENDLY_NAME, builtTable.getFriendlyName()); assertEquals(GENERATED_ID, builtTable.getGeneratedId()); assertEquals(LAST_MODIFIED_TIME, builtTable.getLastModifiedTime()); assertEquals(TABLE_DEFINITION, builtTable.getDefinition()); assertEquals(SELF_LINK, builtTable.getSelfLink()); assertSame(serviceMockReturnsOptions, builtTable.getBigQuery()); }
@Test public void testCopyJob() throws InterruptedException, TimeoutException { String sourceTableName = "test_copy_job_source_table"; String destinationTableName = "test_copy_job_destination_table"; TableId sourceTable = TableId.of(DATASET, sourceTableName); StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition); Table createdTable = bigquery.create(tableInfo); assertNotNull(createdTable); assertEquals(DATASET, createdTable.getTableId().getDataset()); assertEquals(sourceTableName, createdTable.getTableId().getTable()); TableId destinationTable = TableId.of(DATASET, destinationTableName); CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable); Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); Table remoteTable = bigquery.getTable(DATASET, destinationTableName); assertNotNull(remoteTable); assertEquals(destinationTable.getDataset(), remoteTable.getTableId().getDataset()); assertEquals(destinationTableName, remoteTable.getTableId().getTable()); assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); assertTrue(createdTable.delete()); assertTrue(remoteTable.delete()); }
assertNotNull(remoteTable); assertEquals(createdTable.getTableId(), remoteTable.getTableId()); assertTrue(remoteTable.getDefinition() instanceof ViewDefinition); Schema expectedSchema = Schema.of( .setMode(Field.Mode.NULLABLE) .build()); assertEquals(expectedSchema, remoteTable.getDefinition().getSchema()); QueryJobConfiguration config = QueryJobConfiguration.newBuilder("SELECT * FROM " + tableName)
Table remoteTable = bigquery.getTable(DATASET, tableName); assertNotNull(remoteTable); assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition); assertEquals(createdTable.getTableId(), remoteTable.getTableId()); assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); QueryJobConfiguration config = QueryJobConfiguration.newBuilder(
private void checkTableCreation(Table table, FeatureSpec featureSpec) { if (table == null) { throw new StorageInitializationException( Strings.lenientFormat( "Bigquery table creation failed. Possibly linked to BQ rate limiting, please try again later.")); } else { FieldList fields = table.getDefinition().getSchema().getFields(); List<String> fieldNames = fields.stream().map(Field::getName).collect(Collectors.toList()); if (!fieldNames.contains(featureSpec.getName())) { throw new StorageInitializationException( Strings.lenientFormat( "Bigquery table creation failed. Possibly linked to BQ rate limiting, please try again later.")); } } }
Schema existingSchema = table.getDefinition().getSchema(); Field newField = createFeatureField(featureSpec); if (isFieldExist(newField, existingSchema)) {
"Table not found:" + tableId.toString()); bqRowSchema = table.getDefinition().getSchema(); break;
"Table not found:" + tableId.toString()); bqRowSchema = table.getDefinition().getSchema(); break;