@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
Table toPb() { Table tablePb = definition.toPb(); tablePb.setTableReference(tableId.toPb()); if (lastModifiedTime != null) { tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); } tablePb.setCreationTime(creationTime); tablePb.setDescription(description); tablePb.setEtag(etag); tablePb.setExpirationTime(expirationTime); tablePb.setFriendlyName(friendlyName); tablePb.setId(generatedId); tablePb.setSelfLink(selfLink); if (encryptionConfiguration != null) { tablePb.setEncryptionConfiguration(encryptionConfiguration.toPb()); } tablePb.setLabels(labels.toPb()); return tablePb; }
private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, JsonNode node) { if (node.isTextual()) { return new Table() .setTableReference(Bq.tableReference(defaultProjectId, defaultDataset, node.asText())); } else { TableConfig config; try { config = objectMapper.readValue(node.traverse(), TableConfig.class); } catch (IOException e) { throw new ConfigException("Invalid table reference or configuration: " + node, e); } return table(defaultProjectId, defaultDataset, config); } }
public Table toTable(BigQueryTable source) { Table sink = new Table(); TableReference tableRef = tableReference(source); TableSchema schema = toTableSchema(source); sink.setTableReference(tableRef); sink.setSchema(schema); return sink; }
private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, TableConfig config) { Optional<String> datasetId = config.dataset().or(defaultDataset.transform(DatasetReference::getDatasetId)); if (!datasetId.isPresent()) { throw new ConfigException("Bad table reference or configuration: Missing 'dataset'"); } return new Table() .setTableReference(new TableReference() .setProjectId(config.project().or(defaultProjectId)) .setDatasetId(datasetId.get()) .setTableId(config.id())) .setSchema(config.schema().orNull()) .setFriendlyName(config.friendly_name().orNull()) .setExpirationTime(config.expiration_time() .transform(p -> p.getTimestamp().toInstant(request.getTimeZone()).toEpochMilli()).orNull()) .setTimePartitioning(config.time_partitioning().orNull()) .setView(config.view().orNull()); }
Table toPb() { Table tablePb = definition.toPb(); tablePb.setTableReference(tableId.toPb()); if (lastModifiedTime != null) { tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); } tablePb.setCreationTime(creationTime); tablePb.setDescription(description); tablePb.setEtag(etag); tablePb.setExpirationTime(expirationTime); tablePb.setFriendlyName(friendlyName); tablePb.setId(generatedId); tablePb.setSelfLink(selfLink); return tablePb; }
Table toPb() { Table tablePb = definition.toPb(); tablePb.setTableReference(tableId.toPb()); if (lastModifiedTime != null) { tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); } tablePb.setCreationTime(creationTime); tablePb.setDescription(description); tablePb.setEtag(etag); tablePb.setExpirationTime(expirationTime); tablePb.setFriendlyName(friendlyName); tablePb.setId(generatedId); tablePb.setSelfLink(selfLink); return tablePb; }
@Test public void testCreateTableSucceeds() throws IOException { TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table"); Table testTable = new Table().setTableReference(ref); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testTable)); BigQueryServicesImpl.DatasetServiceImpl services = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); Table ret = services.tryCreateTable( testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT); assertEquals(testTable, ret); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
@Test public void testGetTableSucceeds() throws Exception { TableReference tableRef = new TableReference() .setProjectId("projectId") .setDatasetId("datasetId") .setTableId("tableId"); Table testTable = new Table(); testTable.setTableReference(tableRef); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(403).thenReturn(200); when(response.getContent()) .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))) .thenReturn(toStream(testTable)); BigQueryServicesImpl.DatasetServiceImpl datasetService = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); Table table = datasetService.getTable(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT); assertEquals(testTable, table); verify(response, times(2)).getStatusCode(); verify(response, times(2)).getContent(); verify(response, times(2)).getContentType(); }
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
@Test public void testRemoveTemporaryTables() throws Exception { FakeDatasetService datasetService = new FakeDatasetService(); String projectId = "project"; String datasetId = "dataset"; datasetService.createDataset(projectId, datasetId, "", "", null); List<TableReference> tableRefs = Lists.newArrayList( BigQueryHelpers.parseTableSpec( String.format("%s:%s.%s", projectId, datasetId, "table1")), BigQueryHelpers.parseTableSpec( String.format("%s:%s.%s", projectId, datasetId, "table2")), BigQueryHelpers.parseTableSpec( String.format("%s:%s.%s", projectId, datasetId, "table3"))); for (TableReference tableRef : tableRefs) { datasetService.createTable(new Table().setTableReference(tableRef)); } // Add one more table to delete that does not actually exist. tableRefs.add( BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table4"))); WriteRename.removeTemporaryTables(datasetService, tableRefs); for (TableReference ref : tableRefs) { loggedWriteRename.verifyDebug("Deleting table " + toJsonString(ref)); checkState(datasetService.getTable(ref) == null, "Table " + ref + " was not deleted!"); } }
Table toPb() { Table tablePb = definition.toPb(); tablePb.setTableReference(tableId.toPb()); if (lastModifiedTime != null) { tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); } tablePb.setCreationTime(creationTime); tablePb.setDescription(description); tablePb.setEtag(etag); tablePb.setExpirationTime(expirationTime); tablePb.setFriendlyName(friendlyName); tablePb.setId(generatedId); tablePb.setSelfLink(selfLink); if (encryptionConfiguration != null) { tablePb.setEncryptionConfiguration(encryptionConfiguration.toPb()); } tablePb.setLabels(labels.toPb()); return tablePb; }
public static Table createTable(String projectId, String datasetId, String tableId, TableSchema schema, Bigquery bigquery) throws IOException { Table table = new Table(); TableReference tableRef = new TableReference(); tableRef.setDatasetId(datasetId); tableRef.setProjectId(projectId); tableRef.setTableId(tableId); table.setTableReference(tableRef); table.setFriendlyName(tableId); table.setSchema(schema); try { return bigquery.tables().insert(projectId, datasetId, table).execute(); } catch (GoogleJsonResponseException e) { return null; //table already exists } } }
private Table createTable(Description description) throws IOException, InterruptedException { TableReference tableReference = new TableReference() .setProjectId(pipelineOptions.getProject()) .setDatasetId(pipelineOptions.getTargetDataset()) .setTableId(createRandomizedName(description)); table = new Table() .setTableReference(tableReference) .setSchema(BigQueryUtils.toTableSchema(schema)) .setDescription( "Table created for " + description.getDisplayName() + " by TestBigQueryRule. " + "Should be automatically cleaned up after test completion."); if (datasetService.getTable(tableReference) != null) { throw new IllegalStateException( "Table '" + tableReference + "' already exists. " + "It should have been cleaned up by the test rule."); } datasetService.createTable(table); return table; }
/** Tests that table creation succeeds when the table already exists. */ @Test public void testCreateTableSucceedsAlreadyExists() throws IOException { TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table"); TableSchema schema = new TableSchema() .setFields( ImmutableList.of( new TableFieldSchema().setName("column1").setType("String"), new TableFieldSchema().setName("column2").setType("Integer"))); Table testTable = new Table().setTableReference(ref).setSchema(schema); when(response.getStatusCode()).thenReturn(409); // 409 means already exists BigQueryServicesImpl.DatasetServiceImpl services = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); Table ret = services.tryCreateTable( testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT); assertNull(ret); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
private Table toTable(Shape shape, GoogleBigQueryTable dataSource) { Table table = new Table(); BigQueryTableReference ref = dataSource.getTableReference(); TableReference reference = new TableReference(); reference.setProjectId(ref.getProjectId()); reference.setDatasetId(ref.getDatasetId()); reference.setTableId(ref.getTableId()); table.setTableReference(reference); if (!(dataSource instanceof GoogleBigQueryView)){ TableSchema tableSchema = tableGenerator.toTableSchema(shape); table.setSchema(tableSchema); } ExternalDataConfiguration external = dataSource.getExternalDataConfiguration(); if (external != null) { table.setExternalDataConfiguration(external); table.setType("EXTERNAL"); } return table; }
private JobStatus runQueryJob(JobConfigurationQuery query) throws IOException, InterruptedException { List<TableRow> rows = FakeBigQueryServices.rowsFromEncodedQuery(query.getQuery()); datasetService.createTable(new Table().setTableReference(query.getDestinationTable())); datasetService.insertAll(query.getDestinationTable(), rows, null); return new JobStatus().setState("DONE"); }
private void setupNewTypesQueryTest() { this.bqClient.createNewTable( this.project, this.bigQueryDatasetId, new Table() .setSchema(BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_SCHEMA) .setTableReference( new TableReference() .setTableId(BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_NAME) .setDatasetId(this.bigQueryDatasetId) .setProjectId(this.project))); this.bqClient.insertDataToTable( this.project, this.bigQueryDatasetId, BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_NAME, BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_DATA); this.options.setQuery( String.format( "SELECT bytes, date, time FROM [%s:%s.%s]", project, this.bigQueryDatasetId, BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_NAME)); this.options.setOutput(outputTable); this.options.setOutputSchema(BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_SCHEMA); }