/** Example of loading a parquet file from GCS to a table. */ public void loadTableGcsParquet(String datasetName) throws InterruptedException { // [START bigquery_load_table_gcs_parquet] String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"; TableId tableId = TableId.of(datasetName, "us_states"); LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, sourceUri) .setFormatOptions(FormatOptions.parquet()) .build(); // Load the table Job loadJob = bigquery.create(JobInfo.of(configuration)); loadJob = loadJob.waitFor(); // Check the table StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition(); System.out.println("State: " + loadJob.getStatus().getState()); System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows()); // [END bigquery_load_table_gcs_parquet] }
@Test public void testToAndFromPbWithoutConfiguration() { JobInfo jobInfo = JobInfo.newBuilder(null).build(); initializeExpectedJob(4, jobInfo); replay(bigquery); compareJob(expectedJob, Job.fromPb(serviceMockReturnsOptions, expectedJob.toPb())); }
@Override public JobInfo build() { return new JobInfo(this); } }
private void compareJobInfo(JobInfo expected, JobInfo value) { assertEquals(expected, value); assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); assertEquals(expected.getEtag(), value.getEtag()); assertEquals(expected.getGeneratedId(), value.getGeneratedId()); assertEquals(expected.getJobId(), value.getJobId()); assertEquals(expected.getSelfLink(), value.getSelfLink()); assertEquals(expected.getStatus(), value.getStatus()); assertEquals(expected.getStatistics(), value.getStatistics()); assertEquals(expected.getUserEmail(), value.getUserEmail()); assertEquals(expected.getConfiguration(), value.getConfiguration()); } }
@InternalApi("visible for testing") Job create(JobInfo jobInfo, Supplier<JobId> idProvider, JobOption... options) { boolean idRandom = false; if (jobInfo.getJobId() == null) { jobInfo = jobInfo.toBuilder().setJobId(idProvider.get()).build(); idRandom = true; jobInfo.setProjectId(getOptions().getProjectId()).toPb(); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); return Job.fromPb( this, runWithRetries( getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock())); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); job = getJob(jobInfo.getJobId()); } catch (BigQueryException e) { throw createException;
@Test public void testOf() { JobInfo job = JobInfo.of(COPY_CONFIGURATION); assertEquals(COPY_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(EXTRACT_CONFIGURATION); assertEquals(EXTRACT_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(LOAD_CONFIGURATION); assertEquals(LOAD_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(QUERY_CONFIGURATION); assertEquals(QUERY_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, COPY_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(COPY_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, EXTRACT_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(EXTRACT_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, LOAD_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(LOAD_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, QUERY_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(QUERY_CONFIGURATION, job.getConfiguration()); }
@Test public void testGetJobFromJobIdWithProject() { JobId jobId = JobId.of(OTHER_PROJECT, JOB); JobInfo jobInfo = COPY_JOB.setProjectId(OTHER_PROJECT); EasyMock.expect(bigqueryRpcMock.getJob(OTHER_PROJECT, JOB, null, EMPTY_RPC_OPTIONS)) .andReturn(jobInfo.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); Job job = bigquery.getJob(jobId); assertEquals(new Job(bigquery, new JobInfo.BuilderImpl(jobInfo)), job); }
.build(); JobInfo jobInfo = JobInfo.newBuilder(jobConfig) .setJobId(JOB_ID) .setStatistics(jobStatistics) JobStatus status = createStrictMock(JobStatus.class); expect(bigquery.getOptions()).andReturn(mockOptions); expect(mockOptions.getClock()).andReturn(CurrentMillisClock.getDefaultClock()).times(2); Job completedJob = expectedJob.toBuilder().setStatus(status).build(); .build(); expect(bigquery.getQueryResults(jobInfo.getJobId(), Job.DEFAULT_QUERY_WAIT_OPTIONS)) .andReturn(completedQuery); expect(bigquery.getJob(JOB_INFO.getJobId())).andReturn(completedJob); expect(bigquery.getQueryResults(jobInfo.getJobId(), Job.DEFAULT_QUERY_WAIT_OPTIONS)) .andReturn(completedQuery); expect(bigquery.listTableData(eq(TABLE_ID1), anyObject(Schema.class))).andReturn(result); replay(status, bigquery, mockOptions); initializeJob(jobInfo); assertThat(job.waitFor(TEST_RETRY_OPTIONS)).isSameAs(completedJob); assertThat(job.getQueryResults().iterateAll()).hasSize(0); verify(status, mockOptions);
@Test public void testCreateJobSuccess() { String id = "testCreateJobSuccess-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); assertThat(bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query)))).isNotNull(); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
@Test public void testCreateJobTryGet() { final String id = "testCreateJobTryGet-id"; String query = "SELECT * in FOO"; Supplier<JobId> idProvider = new Supplier<JobId>() { @Override public JobId get() { return JobId.of(id); } }; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.expect( bigqueryRpcMock.getJob( anyString(), EasyMock.eq(id), EasyMock.eq((String) null), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); ((BigQueryImpl) bigquery).create(JobInfo.of(QueryJobConfiguration.of(query)), idProvider); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
@Test public void testReload() throws Exception { initializeExpectedJob(4); JobInfo updatedInfo = JOB_INFO.toBuilder().setEtag("etag").build(); Job expectedJob = new Job(serviceMockReturnsOptions, new JobInfo.BuilderImpl(updatedInfo)); expect(bigquery.getOptions()).andReturn(mockOptions); expect(bigquery.getJob(JOB_INFO.getJobId())).andReturn(expectedJob); replay(bigquery); initializeJob(); Job updatedJob = job.reload(); compareJob(expectedJob, updatedJob); }
@Test public void testCreateJobWithSelectedFields() { Capture<Map<BigQueryRpc.Option, Object>> capturedOptions = Capture.newInstance(); EasyMock.expect( bigqueryRpcMock.create( EasyMock.anyObject(com.google.api.services.bigquery.model.Job.class), EasyMock.capture(capturedOptions))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); BigQuery.JobOption jobOptions = BigQuery.JobOption.fields(BigQuery.JobField.USER_EMAIL); bigquery = options.getService(); bigquery.create(JobInfo.of(QueryJobConfiguration.of("SOME QUERY")), jobOptions); String selector = (String) capturedOptions.getValue().get(jobOptions.getRpcOption()); // jobReference and configuration are always sent; the RPC call won't succeed otherwise. assertThat(selector.split(",")) .asList() .containsExactly("jobReference", "configuration", "user_email"); }
@Test public void testCreateJobWithProjectId() { JobInfo jobInfo = JobInfo.newBuilder(QUERY_JOB_CONFIGURATION.setProjectId(OTHER_PROJECT)) .setJobId(JobId.of(OTHER_PROJECT, JOB)) .build(); Capture<Map<BigQueryRpc.Option, Object>> capturedOptions = Capture.newInstance(); EasyMock.expect(bigqueryRpcMock.create(eq(jobInfo.toPb()), capture(capturedOptions))) .andReturn(jobInfo.toPb()); EasyMock.replay(bigqueryRpcMock); BigQueryOptions bigQueryOptions = createBigQueryOptionsForProject(OTHER_PROJECT, rpcFactoryMock); bigquery = bigQueryOptions.getService(); Job job = bigquery.create(jobInfo, JOB_OPTION_FIELDS); assertEquals(new Job(bigquery, new JobInfo.BuilderImpl(jobInfo)), job); String selector = (String) capturedOptions.getValue().get(JOB_OPTION_FIELDS.getRpcOption()); assertTrue(selector.contains("jobReference")); assertTrue(selector.contains("configuration")); assertTrue(selector.contains("user_email")); assertEquals(37, selector.length()); }
.setSchema(TABLE_SCHEMA.toPb()); EasyMock.expect( bigqueryRpcMock.create( JOB_INFO.toPb(), Collections.<BigQueryRpc.Option, Object>emptyMap())) .andReturn(jobResponsePb1); EasyMock.expect(bigqueryRpcMock.getJob(eq(PROJECT), eq(JOB), anyString(), anyObject(Map.class))) .andReturn(jobResponsePb1); EasyMock.expect( bigqueryRpcMock.getQueryResults( PROJECT, JOB, null, BigQueryImpl.optionMap(Job.DEFAULT_QUERY_WAIT_OPTIONS))) .andReturn(responsePb1); EasyMock.expect( bigqueryRpcMock.getQueryResults( EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); TableResult result = bigquery.query(QUERY_JOB_CONFIGURATION_FOR_QUERY, queryJob); assertThat(result.getSchema()).isEqualTo(TABLE_SCHEMA); assertThat(result.getTotalRows()).isEqualTo(1);
.setSchema(TABLE_SCHEMA.toPb()); EasyMock.expect( bigqueryRpcMock.create( JOB_INFO.toPb(), Collections.<BigQueryRpc.Option, Object>emptyMap())) .andReturn(jobResponsePb); optionMap.put(pageSizeOption.getRpcOption(), pageSizeOption.getValue()); EasyMock.expect( bigqueryRpcMock.getQueryResults( PROJECT, JOB, null, BigQueryImpl.optionMap(Job.DEFAULT_QUERY_WAIT_OPTIONS))) .andReturn(responsePb); EasyMock.expect(bigqueryRpcMock.listTableData(PROJECT, DATASET, TABLE, optionMap)) .andReturn( new TableDataList() .setPageToken("") EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); Job job = bigquery.create(JobInfo.of(queryJob, QUERY_JOB_CONFIGURATION_FOR_QUERY)); TableResult result = job.getQueryResults(pageSizeOption); assertThat(result.getSchema()).isEqualTo(TABLE_SCHEMA); assertThat(result.getTotalRows()).isEqualTo(1);
.setSchema(TABLE_SCHEMA.toPb()); EasyMock.expect( bigqueryRpcMock.create( JOB_INFO.toPb(), Collections.<BigQueryRpc.Option, Object>emptyMap())) .andReturn(jobResponsePb); EasyMock.expect( bigqueryRpcMock.getQueryResults( PROJECT, JOB, null, BigQueryImpl.optionMap(Job.DEFAULT_QUERY_WAIT_OPTIONS))) .andReturn(responsePb); EasyMock.expect( bigqueryRpcMock.listTableData( PROJECT, DATASET, TABLE, Collections.<BigQueryRpc.Option, Object>emptyMap())) .andReturn( new TableDataList() .setPageToken("") EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); TableResult result = bigquery.query(QUERY_JOB_CONFIGURATION_FOR_QUERY, queryJob); assertThat(result.getSchema()).isEqualTo(TABLE_SCHEMA); assertThat(result.getTotalRows()).isEqualTo(1);
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition); Table createdTable = bigquery.create(tableInfo); assertNotNull(createdTable); assertEquals(DATASET, createdTable.getTableId().getDataset()); CopyJobConfiguration copyJobConfiguration = CopyJobConfiguration.of(destinationTable, sourceTable); Job createdJob = bigquery.create(JobInfo.of(copyJobConfiguration)); Job remoteJob = bigquery.getJob(createdJob.getJobId()); assertEquals(createdJob.getJobId(), remoteJob.getJobId()); CopyJobConfiguration createdConfiguration = createdJob.getConfiguration(); CopyJobConfiguration remoteConfiguration = remoteJob.getConfiguration();
bigquery.create( DatasetInfo.newBuilder("locationset_" + UUID.randomUUID().toString().replace("-", "_")) .setLocation(location) Schema schema = Schema.of(Field.of("name", LegacySQLTypeName.STRING)); TableDefinition tableDef = StandardTableDefinition.of(schema); Table table = bigquery.create(TableInfo.newBuilder(tableId, tableDef).build()); bigquery.create( JobInfo.of( JobId.newBuilder().setLocation(location).build(), QueryJobConfiguration.of(query))); job = job.waitFor(); assertThat(job.getStatus().getError()).isNull(); assertThat(job.getJobId().getLocation()).isEqualTo(location); JobId jobId = job.getJobId();
/** Example of creating a query job. */ // [TARGET create(JobInfo, JobOption...)] // [VARIABLE "SELECT field FROM my_dataset_name.my_table_name"] public Job createJob(String query) { // [START ] Job job = null; JobConfiguration jobConfiguration = QueryJobConfiguration.of(query); JobInfo jobInfo = JobInfo.of(jobConfiguration); try { job = bigquery.create(jobInfo); } catch (BigQueryException e) { // the job was not created } // [END ] return job; }
.setDestinationTable(destinationTable) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); TableResult result = remoteJob.getQueryResults(); assertEquals(QUERY_RESULT_SCHEMA, result.getSchema()); int rowCount = 0; assertTrue(bigquery.delete(DATASET, tableName)); Job queryJob = bigquery.getJob(remoteJob.getJobId()); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); assertNotNull(statistics.getQueryPlan());