@Override public Job apply(JobList.Jobs jobPb) { JobStatus statusPb = jobPb.getStatus() != null ? jobPb.getStatus() : new JobStatus(); if (statusPb.getState() == null) { statusPb.setState(jobPb.getState()); } if (statusPb.getErrorResult() == null) { statusPb.setErrorResult(jobPb.getErrorResult()); } return new Job() .setConfiguration(jobPb.getConfiguration()) .setId(jobPb.getId()) .setJobReference(jobPb.getJobReference()) .setKind(jobPb.getKind()) .setStatistics(jobPb.getStatistics()) .setStatus(statusPb) .setUserEmail(jobPb.getUserEmail()); } }));
static JobStatus fromPb(com.google.api.services.bigquery.model.JobStatus statusPb) { List<BigQueryError> allErrors = null; if (statusPb.getErrors() != null) { allErrors = Lists.transform(statusPb.getErrors(), BigQueryError.FROM_PB_FUNCTION); } BigQueryError error = statusPb.getErrorResult() != null ? BigQueryError.fromPb(statusPb.getErrorResult()) : null; return new JobStatus(State.valueOf(statusPb.getState()), error, allErrors); } }
com.google.api.services.bigquery.model.JobStatus toPb() { com.google.api.services.bigquery.model.JobStatus statusPb = new com.google.api.services.bigquery.model.JobStatus(); if (state != null) { statusPb.setState(state.toString()); } if (error != null) { statusPb.setErrorResult(error.toPb()); } if (executionErrors != null) { statusPb.setErrors(Lists.transform(executionErrors, BigQueryError.TO_PB_FUNCTION)); } return statusPb; }
.setJobReference(queryJob.toPb()) .setId(JOB) .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")); jobResponsePb.getConfiguration().getQuery().setDestinationTable(TABLE_ID.toPb()); GetQueryResultsResponse responsePb =
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} fails. */ @Test public void testPollJobFailed() throws IOException, InterruptedException { Job testJob = new Job(); testJob.setStatus(new JobStatus().setState("DONE").setErrorResult(new ErrorProto())); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(testJob, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
logger.atFine().log( "Job status (%s ms) %s: %s", elapsedTime, jobReference.getJobId(), pollJob.getStatus().getState()); if (pollJob.getStatus().getState().equals("DONE")) { notDone = false; if (pollJob.getStatus().getErrorResult() != null) { throw new IOException( "Error during BigQuery job execution: " + pollJob.getStatus().getErrorResult());
if (!"FAILED".equals(job.job.getStatus().getState())) { if (numFailures < numFailuresExpected) { ++numFailures; job.job.getStatus().setState("RUNNING"); } else if (job.getJobCount == 2 * GET_JOBS_TRANSITION_INTERVAL + 1) { job.job.setStatus(runJob(job.job)); job.job .getStatus() .setState("FAILED") .setErrorResult( new ErrorProto() .setMessage(
static Status parseStatus(@Nullable Job job) { if (job == null) { return Status.UNKNOWN; } JobStatus status = job.getStatus(); if (status.getErrorResult() != null) { return Status.FAILED; } else if (status.getErrors() != null && !status.getErrors().isEmpty()) { return Status.FAILED; } else { return Status.SUCCEEDED; } }
/** * Polls the status of a BigQuery job, returns Job reference if "Done" * * @param bigquery an authorized BigQuery client * @param projectId a string containing the current project ID * @param jobId a reference to an inserted query Job * @return a reference to the completed Job * @throws IOException * @throws InterruptedException */ private static Job checkQueryResults(Bigquery bigquery, String projectId, JobReference jobId) throws IOException, InterruptedException { // Variables to keep track of total query time long startTime = System.currentTimeMillis(); long elapsedTime; while (true) { Job pollJob = bigquery.jobs().get(projectId, jobId.getJobId()).execute(); elapsedTime = System.currentTimeMillis() - startTime; System.out.format("Job status (%dms) %s: %s\n", elapsedTime, jobId.getJobId(), pollJob.getStatus().getState()); if (pollJob.getStatus().getState().equals("DONE")) { return pollJob; } // Pause execution for one second before polling job status again, to // reduce unnecessary calls to the BigQUery API and lower overall // application bandwidth. Thread.sleep(1000); } } // [END start_query]
@Test public void testGetJobSucceeds() throws Exception { Job testJob = new Job(); testJob.setStatus(new JobStatus()); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(testJob, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
retryId -> { if (retryId.getRetryIndex() < 5) { currentJob.setStatus(new JobStatus().setErrorResult(new ErrorProto())); } else { currentJob.setStatus(new JobStatus().setErrorResult(null));
.setJobReference(queryJob.toPb()) .setId(JOB) .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")); jobResponsePb.getConfiguration().getQuery().setDestinationTable(TABLE_ID.toPb()); GetQueryResultsResponse responsePb =
Table existingTable = datasetService.getTable(destination); if (!validateDispositions(existingTable, createDisposition, writeDisposition)) { return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto()); if (!first) { if (!Objects.equals(partitioning, table.getTimePartitioning())) { return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto()); return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto()); .setTimePartitioning(partitioning)); datasetService.insertAll(destination, allRows, null); return new JobStatus().setState("DONE");
queryConfig.toPrettyString())); JobStatus status = pollJob.getStatus(); if (status.getState().equals("DONE")) { ErrorProto error = status.getErrorResult(); if (error == null) { return pollJob.getConfiguration().getQuery().getDestinationTable();
continue; if ("DONE".equals(status.getState())) { LOG.info("BigQuery job {} completed in state DONE", jobRef); return job;
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} returns UNKNOWN. */ @Test public void testPollJobUnknown() throws IOException, InterruptedException { Job testJob = new Job(); testJob.setStatus(new JobStatus()); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.STOP_BACKOFF); assertEquals(null, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
@Override public Job apply(JobList.Jobs jobPb) { JobStatus statusPb = jobPb.getStatus() != null ? jobPb.getStatus() : new JobStatus(); if (statusPb.getState() == null) { statusPb.setState(jobPb.getState()); } if (statusPb.getErrorResult() == null) { statusPb.setErrorResult(jobPb.getErrorResult()); } return new Job() .setConfiguration(jobPb.getConfiguration()) .setId(jobPb.getId()) .setJobReference(jobPb.getJobReference()) .setKind(jobPb.getKind()) .setStatistics(jobPb.getStatistics()) .setStatus(statusPb) .setUserEmail(jobPb.getUserEmail()); } }));
static JobStatus fromPb(com.google.api.services.bigquery.model.JobStatus statusPb) { List<BigQueryError> allErrors = null; if (statusPb.getErrors() != null) { allErrors = Lists.transform(statusPb.getErrors(), BigQueryError.FROM_PB_FUNCTION); } BigQueryError error = statusPb.getErrorResult() != null ? BigQueryError.fromPb(statusPb.getErrorResult()) : null; return new JobStatus(State.valueOf(statusPb.getState()), error, allErrors); } }
com.google.api.services.bigquery.model.JobStatus toPb() { com.google.api.services.bigquery.model.JobStatus statusPb = new com.google.api.services.bigquery.model.JobStatus(); if (state != null) { statusPb.setState(state.toString()); } if (error != null) { statusPb.setErrorResult(error.toPb()); } if (executionErrors != null) { statusPb.setErrors(Lists.transform(executionErrors, BigQueryError.TO_PB_FUNCTION)); } return statusPb; }
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} succeeds. */ @Test public void testPollJobSucceeds() throws IOException, InterruptedException { Job testJob = new Job(); testJob.setStatus(new JobStatus().setState("DONE")); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(testJob, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }