/** * Discovers all shards of this file. * * <p>Because of eventual consistency, reads may discover no files or fewer files than the shard * template implies. In this case, the read is considered to have failed. */ public List<String> readFilesWithRetries() throws IOException, InterruptedException { return readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); }
/** * Discovers all shards of this file. * * <p>Because of eventual consistency, reads may discover no files or fewer files than the shard * template implies. In this case, the read is considered to have failed. */ public List<String> readFilesWithRetries() throws IOException, InterruptedException { return readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); }
/** Discovers all shards of this file. */ public List<String> readFilesWithRetries() throws IOException, InterruptedException { return readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); }
private static BackOff createBackOff() { return BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff()); } /**
private static BackOff createDefaultBackoff() { return BackOffAdapter.toGcpBackOff(DEFAULT_BACKOFF_FACTORY.backoff()); }
@Override public State getState() { if (terminalState != null) { return terminalState; } return getStateWithRetries( BackOffAdapter.toGcpBackOff(STATUS_BACKOFF_FACTORY.backoff()), Sleeper.DEFAULT); }
/** * Computes a checksum of the sharded file specified in the constructor. Not safe to call until * the writing is complete. */ private String getActualChecksum() { if (actualChecksum == null) { // Load output data List<String> outputs; try { outputs = shardedFile.readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); } catch (Exception e) { throw new RuntimeException(String.format("Failed to read from: %s", shardedFile), e); } // Verify outputs. Checksum is computed using SHA-1 algorithm actualChecksum = computeHash(outputs); LOG.debug("Generated checksum: {}", actualChecksum); } return actualChecksum; }
/** * Returns the project number or throws an exception if the project does not exist or has other * access exceptions. */ private static long getProjectNumber(String projectId, CloudResourceManager crmClient) throws IOException { return getProjectNumber( projectId, crmClient, BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff()), Sleeper.DEFAULT); }
private RunQueryResponse runQueryWithRetries(RunQueryRequest request) throws Exception { Sleeper sleeper = Sleeper.DEFAULT; BackOff backoff = RUNQUERY_BACKOFF.backoff(); while (true) { try { RunQueryResponse response = datastore.runQuery(request); rpcSuccesses.inc(); return response; } catch (DatastoreException exception) { rpcErrors.inc(); if (NON_RETRYABLE_ERRORS.contains(exception.getCode())) { throw exception; } if (!BackOffUtils.next(sleeper, backoff)) { LOG.error("Aborting after {} retries.", MAX_RETRIES); throw exception; } } } }
@Override public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException { BackOff backoff = BackOffAdapter.toGcpBackOff( FluentBackoff.DEFAULT .withMaxRetries(maxAttempts) .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF) .withMaxBackoff(Duration.standardMinutes(1)) .backoff()); return pollJob(jobRef, Sleeper.DEFAULT, backoff); }
PendingJobManager() { this( BackOffAdapter.toGcpBackOff( FluentBackoff.DEFAULT .withMaxRetries(Integer.MAX_VALUE) .withInitialBackoff(Duration.standardSeconds(1)) .withMaxBackoff(Duration.standardMinutes(1)) .backoff())); }
private boolean advanceWithBackoff(UnboundedReader<T> reader, Instant endTime) throws IOException { // Try reading from the source with exponential backoff BackOff backoff = BACKOFF_FACTORY.backoff(); long nextSleep = backoff.nextBackOffMillis(); while (true) { if (nextSleep == BackOff.STOP || (endTime != null && Instant.now().isAfter(endTime))) { return false; } if (reader.advance()) { return true; } Uninterruptibles.sleepUninterruptibly(nextSleep, TimeUnit.MILLISECONDS); nextSleep = backoff.nextBackOffMillis(); } } }
@Test public void testCreateBucket() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.insert(any(String.class), any(Bucket.class))) .thenReturn(mockStorageInsert); when(mockStorageInsert.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new Bucket()); gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper()); }
@Test public void testGetStateReturnsServiceState() throws Exception { Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class); Job statusResponse = new Job(); statusResponse.setCurrentState("JOB_STATE_" + State.RUNNING.name()); when(mockJobs.get(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID))).thenReturn(statusRequest); when(statusRequest.execute()).thenReturn(statusResponse); DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, ImmutableMap.of()); assertEquals( State.RUNNING, job.getStateWithRetries( BackOffAdapter.toGcpBackOff(DataflowPipelineJob.STATUS_BACKOFF_FACTORY.backoff()), fastClock)); }
@Test public void testBucketAccessible() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new Bucket()); assertTrue( gcsUtil.bucketAccessible( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper())); }
@Test public void testGetStateWithExceptionReturnsUnknown() throws Exception { Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class); when(mockJobs.get(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID))).thenReturn(statusRequest); when(statusRequest.execute()).thenThrow(IOException.class); DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, ImmutableMap.of()); long startTime = fastClock.nanoTime(); assertEquals( State.UNKNOWN, job.getStateWithRetries( BackOffAdapter.toGcpBackOff(DataflowPipelineJob.STATUS_BACKOFF_FACTORY.backoff()), fastClock)); long timeDiff = TimeUnit.NANOSECONDS.toMillis(fastClock.nanoTime() - startTime); checkValidInterval( DataflowPipelineJob.STATUS_POLLING_INTERVAL, DataflowPipelineJob.STATUS_POLLING_RETRIES, timeDiff); }
@Test public void testBucketDoesNotExist() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow( googleJsonResponseException( HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see")); assertFalse( gcsUtil.bucketAccessible( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper())); }
/** Tests with bounded interval, custom exponent, limited cumulative time, and a reset. */ @Test public void testBoundedIntervalAndCumTimeWithReset() throws Exception { BackOff backOff = FluentBackoff.DEFAULT .withInitialBackoff(Duration.millis(500)) .withMaxBackoff(Duration.standardSeconds(1)) .withMaxCumulativeBackoff(Duration.standardMinutes(1)) .backoff(); assertThat(countMaximumBackoff(backOff), equalTo(Duration.standardMinutes(1).getMillis())); backOff.reset(); assertThat(countMaximumBackoff(backOff), equalTo(Duration.standardMinutes(1).getMillis())); // sanity check: should get 0 if we don't reset assertThat(countMaximumBackoff(backOff), equalTo(0L)); backOff.reset(); assertThat(countMaximumBackoff(backOff), equalTo(Duration.standardMinutes(1).getMillis())); }
/** Tests with bounded interval, custom exponent, limited retries, and a reset. */ @Test public void testMaxRetriesWithReset() throws Exception { BackOff backOff = FluentBackoff.DEFAULT.withInitialBackoff(Duration.millis(500)).withMaxRetries(1).backoff(); assertThat(backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(249L), lessThan(751L))); assertThat(backOff.nextBackOffMillis(), equalTo(BackOff.STOP)); assertThat(backOff.nextBackOffMillis(), equalTo(BackOff.STOP)); assertThat(backOff.nextBackOffMillis(), equalTo(BackOff.STOP)); assertThat(backOff.nextBackOffMillis(), equalTo(BackOff.STOP)); backOff.reset(); assertThat(backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(249L), lessThan(751L))); assertThat(backOff.nextBackOffMillis(), equalTo(BackOff.STOP)); }
/** Tests with bounded interval, custom exponent, limited cumulative time and retries. */ @Test public void testBoundedIntervalAndCumTimeAndRetriesWithReset() throws Exception { BackOff backOff = FluentBackoff.DEFAULT .withInitialBackoff(Duration.millis(500)) .withMaxBackoff(Duration.standardSeconds(1)) .withMaxCumulativeBackoff(Duration.standardMinutes(1)) .backoff(); long cumulativeBackoffMillis = 0; long currentBackoffMillis = backOff.nextBackOffMillis(); while (currentBackoffMillis != BackOff.STOP) { cumulativeBackoffMillis += currentBackoffMillis; currentBackoffMillis = backOff.nextBackOffMillis(); } assertThat(cumulativeBackoffMillis, equalTo(Duration.standardMinutes(1).getMillis())); }