@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
BuilderImpl(Table tablePb) { this.tableId = TableId.fromPb(tablePb.getTableReference()); if (tablePb.getLastModifiedTime() != null) { this.setLastModifiedTime(tablePb.getLastModifiedTime().longValue()); } this.description = tablePb.getDescription(); this.expirationTime = tablePb.getExpirationTime(); this.friendlyName = tablePb.getFriendlyName(); this.creationTime = tablePb.getCreationTime(); this.etag = tablePb.getEtag(); this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.numBytes = tablePb.getNumBytes(); this.numRows = tablePb.getNumRows(); this.definition = TableDefinition.fromPb(tablePb); if (tablePb.getEncryptionConfiguration() != null) { this.encryptionConfiguration = new EncryptionConfiguration.Builder(tablePb.getEncryptionConfiguration()).build(); } this.labels = Labels.fromPb(tablePb.getLabels()); }
Table toPb() { Table tablePb = definition.toPb(); tablePb.setTableReference(tableId.toPb()); if (lastModifiedTime != null) { tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); } tablePb.setCreationTime(creationTime); tablePb.setDescription(description); tablePb.setEtag(etag); tablePb.setExpirationTime(expirationTime); tablePb.setFriendlyName(friendlyName); tablePb.setId(generatedId); tablePb.setSelfLink(selfLink); if (encryptionConfiguration != null) { tablePb.setEncryptionConfiguration(encryptionConfiguration.toPb()); } tablePb.setLabels(labels.toPb()); return tablePb; }
public Table toTable(BigQueryTable source) { Table sink = new Table(); TableReference tableRef = tableReference(source); TableSchema schema = toTableSchema(source); sink.setTableReference(tableRef); sink.setSchema(schema); return sink; }
private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, TableConfig config) { Optional<String> datasetId = config.dataset().or(defaultDataset.transform(DatasetReference::getDatasetId)); if (!datasetId.isPresent()) { throw new ConfigException("Bad table reference or configuration: Missing 'dataset'"); } return new Table() .setTableReference(new TableReference() .setProjectId(config.project().or(defaultProjectId)) .setDatasetId(datasetId.get()) .setTableId(config.id())) .setSchema(config.schema().orNull()) .setFriendlyName(config.friendly_name().orNull()) .setExpirationTime(config.expiration_time() .transform(p -> p.getTimestamp().toInstant(request.getTimeZone()).toEpochMilli()).orNull()) .setTimePartitioning(config.time_partitioning().orNull()) .setView(config.view().orNull()); }
private void testReadFromTable(boolean useTemplateCompatibility, boolean useReadTableRows) throws IOException, InterruptedException { Table sometable = new Table(); sometable.setSchema( new TableSchema() .setFields( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))); sometable.setTableReference( new TableReference() .setProjectId("non-executing-project") .setDatasetId("somedataset") .setTableId("sometable")); sometable.setNumBytes(1024L * 1024L); FakeDatasetService fakeDatasetService = new FakeDatasetService(); fakeDatasetService.createDataset("non-executing-project", "somedataset", "", "", null); new TableRow().set("name", "b").set("number", 2L), new TableRow().set("name", "c").set("number", 3L)); fakeDatasetService.insertAll(sometable.getTableReference(), records, null);
private Table toTable(Shape shape, GoogleBigQueryTable dataSource) { Table table = new Table(); BigQueryTableReference ref = dataSource.getTableReference(); TableReference reference = new TableReference(); reference.setProjectId(ref.getProjectId()); reference.setDatasetId(ref.getDatasetId()); reference.setTableId(ref.getTableId()); table.setTableReference(reference); if (!(dataSource instanceof GoogleBigQueryView)){ TableSchema tableSchema = tableGenerator.toTableSchema(shape); table.setSchema(tableSchema); } ExternalDataConfiguration external = dataSource.getExternalDataConfiguration(); if (external != null) { table.setExternalDataConfiguration(external); table.setType("EXTERNAL"); } return table; }
private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, JsonNode node) { if (node.isTextual()) { return new Table() .setTableReference(Bq.tableReference(defaultProjectId, defaultDataset, node.asText())); } else { TableConfig config; try { config = objectMapper.readValue(node.traverse(), TableConfig.class); } catch (IOException e) { throw new ConfigException("Invalid table reference or configuration: " + node, e); } return table(defaultProjectId, defaultDataset, config); } }
@SuppressWarnings("unchecked") static StandardTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); if (tablePb.getNumRows() != null) { builder.setNumRows(tablePb.getNumRows().longValue()); } if (tablePb.getStreamingBuffer() != null) { builder.setStreamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); } if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }
Table table = checkNotNull(datasetService.getTable(source)); if (!first) { if (!Objects.equals(partitioning, table.getTimePartitioning())) { return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto()); if (!Objects.equals(schema, table.getSchema())) { return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto()); partitioning = table.getTimePartitioning(); schema = table.getSchema(); first = false; allRows.addAll( new Table() .setTableReference(destination) .setSchema(schema) .setTimePartitioning(partitioning)); datasetService.insertAll(destination, allRows, null); return new JobStatus().setState("DONE");
BuilderImpl(Table tablePb) { this.tableId = TableId.fromPb(tablePb.getTableReference()); if (tablePb.getLastModifiedTime() != null) { this.lastModifiedTime(tablePb.getLastModifiedTime().longValue()); } this.description = tablePb.getDescription(); this.expirationTime = tablePb.getExpirationTime(); this.friendlyName = tablePb.getFriendlyName(); this.creationTime = tablePb.getCreationTime(); this.etag = tablePb.getEtag(); this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.definition = TableDefinition.fromPb(tablePb); }
public static Table createTable(String projectId, String datasetId, String tableId, TableSchema schema, Bigquery bigquery) throws IOException { Table table = new Table(); TableReference tableRef = new TableReference(); tableRef.setDatasetId(datasetId); tableRef.setProjectId(projectId); tableRef.setTableId(tableId); table.setTableReference(tableRef); table.setFriendlyName(tableId); table.setSchema(schema); try { return bigquery.tables().insert(projectId, datasetId, table).execute(); } catch (GoogleJsonResponseException e) { return null; //table already exists } } }
if (table == null) { Table newTable = new Table() .setSchema(schema) .setTableReference( new TableReference() .setProjectId(projectId) .setTableId(tableId)); tableService.insert(projectId, datasetId, newTable).execute(); } else if (!table.getSchema().equals(schema)) { throw new RuntimeException( "Table exists and schemas do not match, expecting: " + schema.toPrettyString() + ", actual: " + table.getSchema().toPrettyString());
private Table createTable(Description description) throws IOException, InterruptedException { TableReference tableReference = new TableReference() .setProjectId(pipelineOptions.getProject()) .setDatasetId(pipelineOptions.getTargetDataset()) .setTableId(createRandomizedName(description)); table = new Table() .setTableReference(tableReference) .setSchema(BigQueryUtils.toTableSchema(schema)) .setDescription( "Table created for " + description.getDisplayName() + " by TestBigQueryRule. " + "Should be automatically cleaned up after test completion."); if (datasetService.getTable(tableReference) != null) { throw new IllegalStateException( "Table '" + tableReference + "' already exists. " + "It should have been cleaned up by the test rule."); } datasetService.createTable(table); return table; }
@Override public Table create(Table table, Map<Option, ?> options) { try { // unset the type, as it is output only table.setType(null); TableReference reference = table.getTableReference(); return bigquery .tables() .insert(reference.getProjectId(), reference.getDatasetId(), table) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
fakeDatasetService.createDataset("project", "data_set", "", "", null); fakeDatasetService.createTable( new Table() .setTableReference(table) .setSchema( new TableSchema() .setFields( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .setStreamingBuffer(new Streamingbuffer().setEstimatedBytes(BigInteger.valueOf(10)))); fakeDatasetService.insertAll(table, data, null);
.clone() .setTableId(BigQueryHelpers.stripPartitionDecorator(destination.getTableId())); existingTable = new Table().setTableReference(strippedDestination).setSchema(schema); if (load.getTimePartitioning() != null) { existingTable = existingTable.setTimePartitioning(load.getTimePartitioning());
public static TableInfo createViewInfo(com.google.api.services.bigquery.model.Table model) { String type = model.getType(); if (type == null) { model.setType(TableDefinition.Type.VIEW.name()); } TableId tableId = TableId.fromPb(model.getTableReference()); return TableInfo.of(tableId, ViewDefinition.of(new String(model.getView().getQuery().getBytes()))); }