TableReference toPb() { return new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table); }
TableSchema schema = new TableSchema(); List<TableFieldSchema> tableFieldSchema = new ArrayList<TableFieldSchema>(); TableFieldSchema schemaEntry = new TableFieldSchema(); schemaEntry.setName(myFirstFieldName); schemaEntry.setType("STRING"); tableFieldSchema.add(schemaEntry); schema.setFields(tableFieldSchema); Table table = new Table(); table.setSchema(schema); TableReference tableRef = new TableReference(); tableRef.setDatasetId(DATASET_ID); tableRef.setProjectId(PROJECT_ID); tableRef.setTableId(tableId); table.setTableReference(tableRef); try { bigquery.tables().insert(PROJECT_ID, DATASET_ID, table).execute(); } catch (IOException e) { }
@Override public TableDestination getTable(Long destination) { return new TableDestination( new TableReference() .setProjectId(writeProject) .setDatasetId(writeDataset) .setTableId(writeTable + "_" + destination), "Table for year " + destination); }
/** * Parse a table specification in the form {@code "[project_id]:[dataset_id].[table_id]"} or * {@code "[dataset_id].[table_id]"}. * * <p>If the project id is omitted, the default project id is used. */ public static TableReference parseTableSpec(String tableSpec) { Matcher match = BigQueryIO.TABLE_SPEC.matcher(tableSpec); if (!match.matches()) { throw new IllegalArgumentException( "Table reference is not in [project_id]:[dataset_id].[table_id] " + "format: " + tableSpec); } TableReference ref = new TableReference(); ref.setProjectId(match.group("PROJECT")); return ref.setDatasetId(match.group("DATASET")).setTableId(match.group("TABLE")); }
/** * Retrieves the fully-qualified TableReference for the desired final destination table for the * output. */ static TableReference getFinalTableReference(Configuration configuration) { String outputProjectId = configuration.get(BigQueryConfiguration.OUTPUT_PROJECT_ID_KEY); String outputDatasetId = configuration.get(BigQueryConfiguration.OUTPUT_DATASET_ID_KEY); String outputTableId = configuration.get(BigQueryConfiguration.OUTPUT_TABLE_ID_KEY); TableReference finalTableRef = new TableReference() .setProjectId(outputProjectId) .setDatasetId(outputDatasetId) .setTableId(outputTableId); return finalTableRef; }
static TableReference createTempTableReference(String projectId, String jobUuid) { String queryTempDatasetId = "temp_dataset_" + jobUuid; String queryTempTableId = "temp_table_" + jobUuid; TableReference queryTempTableRef = new TableReference() .setProjectId(projectId) .setDatasetId(queryTempDatasetId) .setTableId(queryTempTableId); return queryTempTableRef; }
/** Sets the table reference. */ private static TableReference getTableReference(String project, String dataset, String table) { TableReference tableRef = new TableReference(); tableRef.setProjectId(project); tableRef.setDatasetId(dataset); tableRef.setTableId(table); return tableRef; }
/** Utility to construct an output table reference. */ static TableReference getTable(String projectId, String datasetId, String tableName) { TableReference table = new TableReference(); table.setDatasetId(datasetId); table.setProjectId(projectId); table.setTableId(tableName); return table; } }
TableReference toPb() { return new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table); }
TableReference toPb() { return new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table); }
TableReference toPb() { return new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table); }
private TableReference tableReference(BigQueryTable source) { TableReference result = new TableReference(); result.setTableId(source.getTableId()); result.setDatasetId(source.getTableDataset().getDatasetId()); result.setProjectId(source.getTableDataset().getDatasetProject().getProjectId()); return result; }
@Test public void testIsTableEmptyThrows() throws Exception { when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(401); TableReference tableRef = new TableReference() .setProjectId("projectId") .setDatasetId("datasetId") .setTableId("tableId"); BigQueryServicesImpl.DatasetServiceImpl datasetService = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); thrown.expect(IOException.class); thrown.expectMessage(String.format("Unable to list table data: %s", tableRef.getTableId())); datasetService.isTableEmpty(tableRef, BackOff.STOP_BACKOFF, Sleeper.DEFAULT); }
@Test public void testGetTableThrows() throws Exception { when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(401); TableReference tableRef = new TableReference() .setProjectId("projectId") .setDatasetId("datasetId") .setTableId("tableId"); thrown.expect(IOException.class); thrown.expectMessage(String.format("Unable to get table: %s", tableRef.getTableId())); BigQueryServicesImpl.DatasetServiceImpl datasetService = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); datasetService.getTable(tableRef, BackOff.STOP_BACKOFF, Sleeper.DEFAULT); }
public static Table createTable(String projectId, String datasetId, String tableId, TableSchema schema, Bigquery bigquery) throws IOException { Table table = new Table(); TableReference tableRef = new TableReference(); tableRef.setDatasetId(datasetId); tableRef.setProjectId(projectId); tableRef.setTableId(tableId); table.setTableReference(tableRef); table.setFriendlyName(tableId); table.setSchema(schema); try { return bigquery.tables().insert(projectId, datasetId, table).execute(); } catch (GoogleJsonResponseException e) { return null; //table already exists } } }
@Test public void testBuildSourceWithTableReference() { TableReference table = new TableReference() .setProjectId("foo.com:project") .setDatasetId("somedataset") .setTableId("sometable"); BigQueryIO.Read read = BigQueryIO.read().from(table); checkReadTableObject(read, "foo.com:project", "somedataset", "sometable"); }
@Override public PDone expand(PCollection<IndexedRecord> in) { TableReference table = new TableReference(); table.setProjectId(datastore.projectName.getValue()); table.setDatasetId(dataset.bqDataset.getValue()); table.setTableId(dataset.tableName.getValue()); BigQueryIO.Write bigQueryIOPTransform = BigQueryIO.writeTableRows().to(table); bigQueryIOPTransform = setTableOperation(bigQueryIOPTransform); bigQueryIOPTransform = setWriteOperation(bigQueryIOPTransform); in.apply(ParDo.of(new IndexedRecordToTableRowFn())).apply(bigQueryIOPTransform); return PDone.in(in.getPipeline()); }
@Override public PDone expand(PCollection<IndexedRecord> in) { TableReference table = new TableReference(); table.setProjectId(datastore.projectName.getValue()); table.setDatasetId(dataset.bqDataset.getValue()); table.setTableId(dataset.tableName.getValue()); BigQueryIO.Write bigQueryIOPTransform = BigQueryIO.writeTableRows().to(table); bigQueryIOPTransform = setTableOperation(bigQueryIOPTransform); bigQueryIOPTransform = setWriteOperation(bigQueryIOPTransform); in.apply(ParDo.of(new IndexedRecordToTableRowFn())).apply(bigQueryIOPTransform); return PDone.in(in.getPipeline()); }
@Test public void testTableGet() throws InterruptedException, IOException { onTableGet(basicTableSchema()); TableDataList dataList = new TableDataList().setTotalRows(0L); onTableList(dataList); BigQueryServicesImpl.DatasetServiceImpl services = new BigQueryServicesImpl.DatasetServiceImpl(mockClient, options); services.getTable( new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table")); verifyTableGet(); }
@Test public void testBuildWriteWithTableReference() { TableReference table = new TableReference() .setProjectId("foo.com:project") .setDatasetId("somedataset") .setTableId("sometable"); BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows().to(table); assertEquals("foo.com:project", write.getTable().get().getProjectId()); assertEquals("somedataset", write.getTable().get().getDatasetId()); assertEquals("sometable", write.getTable().get().getTableId()); }