@Override void run(BigQuery bigquery, JobInfo job) throws Exception { System.out.println("Creating job"); Job startedJob = bigquery.create(job); while (!startedJob.isDone()) { System.out.println("Waiting for job " + startedJob.getJobId().getJob() + " to complete"); Thread.sleep(1000L); } startedJob = startedJob.reload(); if (startedJob.getStatus().getError() == null) { System.out.println("Job " + startedJob.getJobId().getJob() + " succeeded"); } else { System.out.println("Job " + startedJob.getJobId().getJob() + " failed"); System.out.println("Error: " + startedJob.getStatus().getError()); } } }
/** Example usage of {@code waitFor()}. */ // [TARGET waitFor(RetryOption...)] public boolean waitFor() throws InterruptedException { try { // [START ] Job completedJob = job.waitFor(); if (completedJob == null) { // job no longer exists } else if (completedJob.getStatus().getError() != null) { // job failed, handle error } else { // job completed successfully } // [END ] } catch (BigQueryException e) { // Timeouts shouldn't happen without a timeout option. if (e.getCause() instanceof PollException) { return false; } throw e; } return true; }
/** Example of partitioning data to a list of Google Cloud Storage files. */ // [TARGET extract(String, List, JobOption...)] // [VARIABLE "CSV"] // [VARIABLE "gs://my_bucket/PartitionA_*.csv"] // [VARIABLE "gs://my_bucket/PartitionB_*.csv"] public Job extractList(String format, String gcsUrl1, String gcsUrl2) { // [START ] List<String> destinationUris = new ArrayList<>(); destinationUris.add(gcsUrl1); destinationUris.add(gcsUrl2); Job job = table.extract(format, destinationUris); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
/** Example loading data from a list of Google Cloud Storage files. */ // [TARGET load(FormatOptions, List, JobOption...)] // [VARIABLE "gs://my_bucket/filename1.csv"] // [VARIABLE "gs://my_bucket/filename2.csv"] public Job loadList(String gcsUrl1, String gcsUrl2) { // [START ] List<String> sourceUris = new ArrayList<>(); sourceUris.add(gcsUrl1); sourceUris.add(gcsUrl2); Job job = table.load(FormatOptions.csv(), sourceUris); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
public static void main(String... args) throws InterruptedException, TimeoutException { BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId tableId = TableId.of("dataset", "table"); Table table = bigquery.getTable(tableId); if (table == null) { System.out.println("Creating table " + tableId); Field integerField = Field.of("fieldName", LegacySQLTypeName.INTEGER); Schema schema = Schema.of(integerField); table = bigquery.create(TableInfo.of(tableId, StandardTableDefinition.of(schema))); } System.out.println("Loading data into table " + tableId); Job loadJob = table.load(FormatOptions.csv(), "gs://bucket/path"); loadJob = loadJob.waitFor(); if (loadJob.getStatus().getError() != null) { System.out.println("Job completed with errors"); } else { System.out.println("Job succeeded"); } } }
/** Example extracting data to single Google Cloud Storage file. */ // [TARGET extract(String, String, JobOption...)] // [VARIABLE "CSV"] // [VARIABLE "gs://my_bucket/filename.csv"] public Job extractSingle(String format, String gcsUrl) { // [START bigquery_extract_table] Job job = table.extract(format, gcsUrl); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_extract_table] return job; }
/** Example of copying the table to a destination table. */ // [TARGET copy(String, String, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copy(String datasetName, String tableName) { // [START ] Job job = table.copy(datasetName, tableName); // Wait for the job to complete. try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
/** Example usage of {@code waitFor()} with checking period and timeout. */ // [TARGET waitFor(RetryOption...)] public boolean waitForWithOptions() throws InterruptedException { try { // [START ] Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(1))); if (completedJob == null) { // job no longer exists } else if (completedJob.getStatus().getError() != null) { // job failed, handle error } else { // job completed successfully } // [END ] } catch (BigQueryException e) { if (e.getCause() instanceof PollException) { return false; } throw e; } return true; }
job = reload(); if (job.getStatus() != null && job.getStatus().getError() != null) { throw new JobException( getJobId(), ImmutableList.copyOf(job.getStatus().getExecutionErrors()));
/** Example loading data from a single Google Cloud Storage file. */ // [TARGET load(FormatOptions, String, JobOption...)] // [VARIABLE "gs://my_bucket/filename.csv"] public Job loadSingle(String sourceUri) { // [START bigquery_load_table_gcs_csv] Job job = table.load(FormatOptions.csv(), sourceUri); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_load_table_gcs_csv] return job; } }
@Test public void testConstructor() { assertEquals(STATE, JOB_STATUS.getState()); assertEquals(ERROR, JOB_STATUS.getError()); assertEquals(ALL_ERRORS, JOB_STATUS.getExecutionErrors()); assertEquals(STATE, JOB_STATUS_INCOMPLETE1.getState()); assertEquals(ERROR, JOB_STATUS_INCOMPLETE1.getError()); assertEquals(null, JOB_STATUS_INCOMPLETE1.getExecutionErrors()); assertEquals(STATE, JOB_STATUS_INCOMPLETE2.getState()); assertEquals(null, JOB_STATUS_INCOMPLETE2.getError()); assertEquals(null, JOB_STATUS_INCOMPLETE2.getExecutionErrors()); }
/** Example copying the table to a destination table. */ // [TARGET copy(TableId, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copyTableId(String dataset, String tableName) throws BigQueryException { // [START bigquery_copy_table] TableId destinationId = TableId.of(dataset, tableName); JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); Job job = table.copy(destinationId, options); // Wait for the job to complete. try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully. } else { // Handle error case. } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_copy_table] return job; }
@Test public void testExtractJob() throws InterruptedException, TimeoutException { String tableName = "test_export_job_table"; TableId destinationTable = TableId.of(DATASET, tableName); LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + LOAD_FILE) .setSchema(SIMPLE_SCHEMA) .build(); Job remoteLoadJob = bigquery.create(JobInfo.of(configuration)); remoteLoadJob = remoteLoadJob.waitFor(); assertNull(remoteLoadJob.getStatus().getError()); ExtractJobConfiguration extractConfiguration = ExtractJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE) .setPrintHeader(false) .build(); Job remoteExtractJob = bigquery.create(JobInfo.of(extractConfiguration)); remoteExtractJob = remoteExtractJob.waitFor(); assertNull(remoteExtractJob.getStatus().getError()); String extractedCsv = new String(storage.readAllBytes(BUCKET, EXTRACT_FILE), StandardCharsets.UTF_8); assertEquals( Sets.newHashSet(CSV_CONTENT.split("\n")), Sets.newHashSet(extractedCsv.split("\n"))); assertTrue(bigquery.delete(DATASET, tableName)); }
@Test public void testCancelJob() throws InterruptedException, TimeoutException { String destinationTableName = "test_cancel_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, destinationTableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertTrue(remoteJob.cancel()); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); }
@BeforeClass public static void beforeClass() throws InterruptedException, TimeoutException { RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); RemoteStorageHelper storageHelper = RemoteStorageHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); storage = storageHelper.getOptions().getService(); storage.create(BucketInfo.of(BUCKET)); storage.create( BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(), CSV_CONTENT.getBytes(StandardCharsets.UTF_8)); storage.create( BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(), JSON_CONTENT.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); bigquery.create(info); LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder( TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .setSchema(TABLE_SCHEMA) .build(); Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); assertNull(job.getStatus().getError()); }
@Test public void testCopyJob() throws InterruptedException, TimeoutException { String sourceTableName = "test_copy_job_source_table"; String destinationTableName = "test_copy_job_destination_table"; TableId sourceTable = TableId.of(DATASET, sourceTableName); StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition); Table createdTable = bigquery.create(tableInfo); assertNotNull(createdTable); assertEquals(DATASET, createdTable.getTableId().getDataset()); assertEquals(sourceTableName, createdTable.getTableId().getTable()); TableId destinationTable = TableId.of(DATASET, destinationTableName); CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable); Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); Table remoteTable = bigquery.getTable(DATASET, destinationTableName); assertNotNull(remoteTable); assertEquals(destinationTable.getDataset(), remoteTable.getTableId().getDataset()); assertEquals(destinationTableName, remoteTable.getTableId().getTable()); assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); assertTrue(createdTable.delete()); assertTrue(remoteTable.delete()); }
Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError());
assertNull(completedJob.getStatus().getError()); assertTrue(bigquery.delete(DATASET, destinationTableName));
RetryOption.totalTimeout(Duration.ofMinutes(1))); assertNotNull(completedJob); assertNull(completedJob.getStatus().getError()); assertTrue(bigquery.delete(DATASET, destinationTableName));