@Override public Job create(Job job, Map<Option, ?> options) { try { String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() : this.options.getProjectId(); return bigquery .jobs() .insert(projectId, job) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb); } this.userEmail = jobPb.getUserEmail(); if (jobPb.getConfiguration() != null) { this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); } }
@Test public void testCreateJobSuccess() { String id = "testCreateJobSuccess-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); assertThat(bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query)))).isNotNull(); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
@Test public void testCreateJobNoGet() { String id = "testCreateJobNoGet-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); try { bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query))); fail("should throw"); } catch (BigQueryException e) { assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); } }
@Test public void testCreateJobTryGet() { final String id = "testCreateJobTryGet-id"; String query = "SELECT * in FOO"; Supplier<JobId> idProvider = new Supplier<JobId>() { @Override public JobId get() { return JobId.of(id); } }; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.expect( bigqueryRpcMock.getJob( anyString(), EasyMock.eq(id), EasyMock.eq((String) null), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); ((BigQueryImpl) bigquery).create(JobInfo.of(QueryJobConfiguration.of(query)), idProvider); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
/** * Helper to check for non-null Job.getJobReference().getJobId() and quality of the getJobId() * between {@code expected} and {@code actual}, using Preconditions.checkState. */ public void checkJobIdEquality(Job expected, Job actual) { Preconditions.checkState(actual.getJobReference() != null && actual.getJobReference().getJobId() != null && expected.getJobReference() != null && expected.getJobReference().getJobId() != null && actual.getJobReference().getJobId().equals(expected.getJobReference().getJobId()), "jobIds must match in '[expected|actual].getJobReference()' (got '%s' vs '%s')", expected.getJobReference(), actual.getJobReference()); }
@Override public Job create(Job job, Map<Option, ?> options) { try { String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() : this.options.getProjectId(); return bigquery .jobs() .insert(projectId, job) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
job.getJobReference() != null && job.getJobReference().getJobId() != null, "Require non-null JobReference and JobId inside; getJobReference() == '%s'", job.getJobReference()); Insert insert = service.jobs().insert(projectId, job); Job response = null; logger.atInfo().withCause(ioe).log( "Fetching existing job after catching exception for duplicate jobId '%s'", job.getJobReference().getJobId()); response = service.jobs().get(projectId, job.getJobReference().getJobId()).execute(); } else { throw new IOException(
/** * Makes an API call to the BigQuery API * * @param bigquery an authorized BigQuery client * @param projectId a string containing the current project ID * @param completedJob to the completed Job * @throws IOException */ private static void displayQueryResults(Bigquery bigquery, String projectId, Job completedJob) throws IOException { GetQueryResultsResponse queryResult = bigquery.jobs() .getQueryResults( projectId, completedJob .getJobReference() .getJobId() ).execute(); List<TableRow> rows = queryResult.getRows(); System.out.print("\nQuery Results:\n------------\n"); for (TableRow row : rows) { for (TableCell field : row.getF()) { System.out.printf("%-50s", field.getV()); } System.out.println(); } } // [END display_result]
BackOff backoff) throws IOException, InterruptedException { JobReference jobRef = job.getJobReference(); Exception lastException; do {
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb.getStatistics()); } this.userEmail = jobPb.getUserEmail(); this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); }
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb.getStatistics()); } this.userEmail = jobPb.getUserEmail(); this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); }
private List<ResourceId> executeExtract( String jobId, TableReference table, JobService jobService, String executingProject, String extractDestinationDir, String bqLocation) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setProjectId(executingProject).setLocation(bqLocation).setJobId(jobId); String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir); JobConfigurationExtract extract = new JobConfigurationExtract() .setSourceTable(table) .setDestinationFormat("AVRO") .setDestinationUris(ImmutableList.of(destinationUri)); LOG.info("Starting BigQuery extract job: {}", jobId); jobService.startExtractJob(jobRef, extract); Job extractJob = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES); if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) { throw new IOException( String.format( "Extract job %s failed, status: %s.", extractJob.getJobReference().getJobId(), BigQueryHelpers.statusToPrettyString(extractJob.getStatus()))); } LOG.info("BigQuery extract job completed: {}", jobId); return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob); }
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb); } this.userEmail = jobPb.getUserEmail(); if (jobPb.getConfiguration() != null) { this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); } }
/** * Creates a Query Job for a particular query on a dataset * * @param bigquery an authorized BigQuery client * @param projectId a String containing the project ID * @param querySql the actual query string * @return a reference to the inserted query job * @throws IOException */ public static JobReference startQuery(Bigquery bigquery, String projectId, String querySql) throws IOException { System.out.format("\nInserting Query Job: %s\n", querySql); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationQuery queryConfig = new JobConfigurationQuery(); config.setQuery(queryConfig); job.setConfiguration(config); queryConfig.setQuery(querySql); Insert insert = bigquery.jobs().insert(projectId, job); insert.setProjectId(projectId); JobReference jobId = insert.execute().getJobReference(); System.out.format("\nJob ID of Query Job is: %s\n", jobId.getJobId()); return jobId; }
String.format("Error when trying to execute the job for query %s.", queryConfig.toPrettyString())); JobReference jobId = queryJob.getJobReference();
bqClient .jobs() .getQueryResults(projectId, insertedJob.getJobReference().getJobId()) .execute();
@Override public void beginExport() throws IOException { // Create job and configuration. JobConfigurationExtract extractConfig = new JobConfigurationExtract(); // Set source. extractConfig.setSourceTable(tableToExport.getTableReference()); // Set destination. extractConfig.setDestinationUris(getExportPaths()); extractConfig.set(DESTINATION_FORMAT_KEY, fileFormat.getFormatIdentifier()); JobConfiguration config = new JobConfiguration(); config.setExtract(extractConfig); JobReference jobReference = bigQueryHelper.createJobReference( projectId, "exporttocloudstorage", tableToExport.getLocation()); Job job = new Job(); job.setConfiguration(config); job.setJobReference(jobReference); // Insert and run job. try { Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job); logger.atFine().log("Got response '%s'", response); exportJobReference = response.getJobReference(); } catch (IOException e) { String error = String.format( "Error while exporting table %s", BigQueryStrings.toString(tableToExport.getTableReference())); throw new IOException(error, e); } }
}, retryId -> { if (retryId.getJobId().equals(currentJob.getJobReference().getJobId())) { return currentJob; } else {
private JobStatus runJob(Job job) throws InterruptedException, IOException { if (job.getConfiguration().getLoad() != null) { return runLoadJob(job.getJobReference(), job.getConfiguration().getLoad()); } else if (job.getConfiguration().getCopy() != null) { return runCopyJob(job.getConfiguration().getCopy()); } else if (job.getConfiguration().getExtract() != null) { return runExtractJob(job, job.getConfiguration().getExtract()); } else if (job.getConfiguration().getQuery() != null) { return runQueryJob(job.getConfiguration().getQuery()); } return new JobStatus().setState("DONE"); }