TableReference toPb() { return new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table); }
static TableId fromPb(TableReference tableRef) { return new TableId(tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()); } }
@Override public Table create(Table table, Map<Option, ?> options) { try { // unset the type, as it is output only table.setType(null); TableReference reference = table.getTableReference(); return bigquery .tables() .insert(reference.getProjectId(), reference.getDatasetId(), table) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
@Test public void testBuildWriteWithTableReference() { TableReference table = new TableReference() .setProjectId("foo.com:project") .setDatasetId("somedataset") .setTableId("sometable"); BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows().to(table); assertEquals("foo.com:project", write.getTable().get().getProjectId()); assertEquals("somedataset", write.getTable().get().getDatasetId()); assertEquals("sometable", write.getTable().get().getTableId()); }
public static void main(String[] args) { Options options = getOptions(args); String projectId = options.getProject(); String datasetId = options.getInputBigQueryDataset(); String tableId = options.getInputBigQueryTable(); String namespace = options.getOutputDatastoreNamespace(); String kind = options.getOutputDatastoreKind(); String keyColumn = options.getKeyColumn(); LinkedHashMap<String, String> parents = parseParentPaths(options.getParentPaths()); List<String> indexedColumns = parseIndexedColumns(options.getIndexedColumns()); // Input TableReference tableRef = new TableReference().setDatasetId(datasetId).setTableId(tableId); BigQueryIO.Read reader = BigQueryIO.read().from(tableRef); // Output DatastoreV1.Write writer = DatastoreIO.v1().write().withProjectId(projectId); // Build and run pipeline TableRow2EntityFn fn = new TableRow2EntityFn(projectId, namespace, parents, kind, keyColumn, indexedColumns); Pipeline pipeline = Pipeline.create(options); pipeline .apply(reader) .apply(ParDo.of(fn)) .apply(writer); pipeline.run(); }
DatasetInfo setProjectId(String projectId) { Builder builder = toBuilder(); builder.setDatasetId(getDatasetId().setProjectId(projectId)); if (getAcl() != null) { List<Acl> acls = Lists.newArrayListWithCapacity(getAcl().size()); for (Acl acl : getAcl()) { if (acl.getEntity().getType() == Acl.Entity.Type.VIEW) { Dataset.Access accessPb = acl.toPb(); TableReference viewReferencePb = accessPb.getView(); if (viewReferencePb.getProjectId() == null) { viewReferencePb.setProjectId(projectId); } acls.add(Acl.of(new Acl.View(TableId.fromPb(viewReferencePb)))); } else { acls.add(acl); } } builder.setAcl(acls); } return builder.build(); }
void createTable(String projectId, Table table) throws IOException { String datasetId = table.getTableReference().getDatasetId(); try { client.tables().insert(projectId, datasetId, table) .execute(); } catch (GoogleJsonResponseException e) { if (e.getStatusCode() == HttpStatusCodes.STATUS_CODE_CONFLICT) { logger.debug("Table already exists: {}:{}.{}", projectId, datasetId, table.getTableReference().getTableId()); } else { throw e; } } }
@FinishBundle public void finishBundle(FinishBundleContext c) throws Exception { DatasetService datasetService = bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)); PendingJobManager jobManager = new PendingJobManager(); for (PendingJobData pendingJob : pendingJobs) { jobManager.addPendingJob( pendingJob.retryJob, j -> { try { if (pendingJob.tableDestination.getTableDescription() != null) { TableReference ref = pendingJob.tableDestination.getTableReference(); datasetService.patchTableDescription( ref.clone() .setTableId(BigQueryHelpers.stripPartitionDecorator(ref.getTableId())), pendingJob.tableDestination.getTableDescription()); } removeTemporaryTables(datasetService, pendingJob.tempTables); return null; } catch (IOException | InterruptedException e) { return e; } }); } jobManager.waitForDone(); }
/** * Validates a table reference for whole-table operations, such as create/delete/patch. Such * operations do not support partition decorators. */ private static void validateWholeTableReference(TableReference tableReference) throws IOException { final Pattern tableRegexp = Pattern.compile("[-\\w]{1,1024}"); if (!tableRegexp.matcher(tableReference.getTableId()).matches()) { throw new IOException( String.format( "invalid table ID %s. Table IDs must be alphanumeric " + "(plus underscores) and must be at most 1024 characters long. Also, table" + " decorators cannot be used.", tableReference.getTableId())); } }
@Override public Table create(Table table, Map<Option, ?> options) { try { // unset the type, as it is output only table.setType(null); return bigquery.tables() .insert(this.options.projectId(), table.getTableReference().getDatasetId(), table) .setFields(FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
/** * Constructs a {@code PatchedBigQueryTableRowIterator} that reads from the specified table. */ public static PatchedBigQueryTableRowIterator fromTable(TableReference ref, Bigquery client) { checkNotNull(ref, "ref"); checkNotNull(client, "client"); return new PatchedBigQueryTableRowIterator(ref, /* queryConfig */null, ref.getProjectId(), client); }
@Test public void testCreateNeverWithStreaming() throws Exception { p.enableAbandonedNodeEnforcement(false); TableReference tableRef = new TableReference(); tableRef.setDatasetId("dataset"); tableRef.setTableId("sometable"); PCollection<TableRow> tableRows = p.apply(GenerateSequence.from(0)) .apply( MapElements.via( new SimpleFunction<Long, TableRow>() { @Override public TableRow apply(Long input) { return null; } })) .setCoder(TableRowJsonCoder.of()); tableRows.apply( BigQueryIO.writeTableRows() .to(tableRef) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) .withoutValidation()); }
/** * Sets the {@link TableReference#projectId} of the provided table reference to the id of the * default project if the table reference does not have a project ID specified. */ private TableReference setDefaultProjectIfAbsent( BigQueryOptions bqOptions, TableReference tableReference) { if (Strings.isNullOrEmpty(tableReference.getProjectId())) { checkState( !Strings.isNullOrEmpty(bqOptions.getProject()), "No project ID set in %s or %s, cannot construct a complete %s", TableReference.class.getSimpleName(), BigQueryOptions.class.getSimpleName(), TableReference.class.getSimpleName()); LOG.info( "Project ID not set in {}. Using default project from {}.", TableReference.class.getSimpleName(), BigQueryOptions.class.getSimpleName()); tableReference.setProjectId(bqOptions.getProject()); } return tableReference; }
@Override protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception { TableReference tableToRemove = createTempTableReference( bqOptions.getProject(), createJobIdToken(bqOptions.getJobName(), stepUuid)); DatasetService tableService = bqServices.getDatasetService(bqOptions); LOG.info("Deleting temporary table with query results {}", tableToRemove); tableService.deleteTable(tableToRemove); LOG.info("Deleting temporary dataset with query results {}", tableToRemove.getDatasetId()); tableService.deleteDataset(tableToRemove.getProjectId(), tableToRemove.getDatasetId()); }
@Override public Table patch(Table table, Map<Option, ?> options) { try { // unset the type, as it is output only table.setType(null); TableReference reference = table.getTableReference(); return bigquery.tables() .patch(this.options.projectId(), reference.getDatasetId(), reference.getTableId(), table) .setFields(FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
public void deleteDataset(String projectId, String datasetId) { try { TableList tables = bqClient.tables().list(projectId, datasetId).execute(); for (Tables table : tables.getTables()) { this.deleteTable(projectId, datasetId, table.getTableReference().getTableId()); } } catch (Exception e) { LOG.debug("Exceptions caught when listing all tables: " + e.getMessage()); } try { bqClient.datasets().delete(projectId, datasetId).execute(); LOG.info("Successfully deleted dataset: " + datasetId); } catch (Exception e) { LOG.debug("Exceptions caught when deleting dataset: " + e.getMessage()); } }
@Override public Table create(Table table, Map<Option, ?> options) { try { // unset the type, as it is output only table.setType(null); return bigquery.tables() .insert(this.options.projectId(), table.getTableReference().getDatasetId(), table) .setFields(FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
TableSchema schema = new TableSchema(); List<TableFieldSchema> tableFieldSchema = new ArrayList<TableFieldSchema>(); TableFieldSchema schemaEntry = new TableFieldSchema(); schemaEntry.setName(myFirstFieldName); schemaEntry.setType("STRING"); tableFieldSchema.add(schemaEntry); schema.setFields(tableFieldSchema); Table table = new Table(); table.setSchema(schema); TableReference tableRef = new TableReference(); tableRef.setDatasetId(DATASET_ID); tableRef.setProjectId(PROJECT_ID); tableRef.setTableId(tableId); table.setTableReference(tableRef); try { bigquery.tables().insert(PROJECT_ID, DATASET_ID, table).execute(); } catch (IOException e) { }
@Override public Table patch(Table table, Map<Option, ?> options) { try { // unset the type, as it is output only table.setType(null); TableReference reference = table.getTableReference(); return bigquery .tables() .patch(reference.getProjectId(), reference.getDatasetId(), reference.getTableId(), table) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
private void testWriteValidatesDataset(boolean unbounded) throws Exception { TableReference tableRef = new TableReference(); tableRef.setDatasetId("somedataset"); tableRef.setTableId("sometable");