/** * Discovers all shards of this file. * * <p>Because of eventual consistency, reads may discover no files or fewer files than the shard * template implies. In this case, the read is considered to have failed. */ public List<String> readFilesWithRetries() throws IOException, InterruptedException { return readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); }
PendingJobManager() { this( BackOffAdapter.toGcpBackOff( FluentBackoff.DEFAULT .withMaxRetries(Integer.MAX_VALUE) .withInitialBackoff(Duration.standardSeconds(1)) .withMaxBackoff(Duration.standardMinutes(1)) .backoff())); }
@Test public void testFluentBackoffToString() throws IOException { FluentBackoff config = FluentBackoff.DEFAULT .withExponent(3.4) .withMaxRetries(4) .withInitialBackoff(Duration.standardSeconds(3)) .withMaxBackoff(Duration.standardHours(1)) .withMaxCumulativeBackoff(Duration.standardDays(1)); assertEquals( "FluentBackoff{exponent=3.4, initialBackoff=PT3S, maxBackoff=PT3600S," + " maxRetries=4, maxCumulativeBackoff=PT86400S}", config.toString()); }
public static <T> T executeWithBackOff(AbstractGoogleClientRequest<T> client, String error) throws IOException, InterruptedException { Sleeper sleeper = Sleeper.DEFAULT; BackOff backOff = BackOffAdapter.toGcpBackOff( FluentBackoff.DEFAULT .withMaxRetries(MAX_RETRIES).withInitialBackoff(INITIAL_BACKOFF_TIME).backoff()); T result = null; while (true) { try { result = client.execute(); break; } catch (IOException e) { LOG.error("{}", error, e); if (!BackOffUtils.next(sleeper, backOff)) { String errorMessage = String.format( "%s Failing to execute job after %d attempts.", error, MAX_RETRIES + 1); LOG.error("{}", errorMessage, e); throw new IOException(errorMessage, e); } } } return result; }
@Setup public void setup() throws IOException { ConnectionConfiguration connectionConfiguration = spec.getConnectionConfiguration(); backendVersion = getBackendVersion(connectionConfiguration); restClient = connectionConfiguration.createClient(); retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(0).withInitialBackoff(RETRY_INITIAL_BACKOFF); if (spec.getRetryConfiguration() != null) { retryBackoff = FluentBackoff.DEFAULT .withInitialBackoff(RETRY_INITIAL_BACKOFF) .withMaxRetries(spec.getRetryConfiguration().getMaxAttempts() - 1) .withMaxCumulativeBackoff(spec.getRetryConfiguration().getMaxDuration()); } }
.withInitialBackoff(MESSAGES_POLLING_INTERVAL) .withMaxRetries(MESSAGES_POLLING_RETRIES) .withExponent(DEFAULT_BACKOFF_EXPONENT); protected static final FluentBackoff STATUS_BACKOFF_FACTORY = FluentBackoff.DEFAULT .withInitialBackoff(STATUS_POLLING_INTERVAL) .withMaxRetries(STATUS_POLLING_RETRIES) .withExponent(DEFAULT_BACKOFF_EXPONENT); backoff = BackOffAdapter.toGcpBackOff(MESSAGES_BACKOFF_FACTORY.backoff()); } else { backoff = BackOffAdapter.toGcpBackOff( MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(duration).backoff()); BackOffAdapter.toGcpBackOff(STATUS_BACKOFF_FACTORY.withMaxRetries(0).backoff()), sleeper); boolean hasError = state == State.UNKNOWN; backoff = BackOffAdapter.toGcpBackOff( MESSAGES_BACKOFF_FACTORY.withMaxCumulativeBackoff(remaining).backoff()); } else {
/** Tests with bounded interval, custom exponent, limited cumulative time and retries. */ @Test public void testBoundedIntervalAndCumTimeAndRetriesWithReset() throws Exception { BackOff backOff = FluentBackoff.DEFAULT .withInitialBackoff(Duration.millis(500)) .withMaxBackoff(Duration.standardSeconds(1)) .withMaxCumulativeBackoff(Duration.standardMinutes(1)) .backoff(); long cumulativeBackoffMillis = 0; long currentBackoffMillis = backOff.nextBackOffMillis(); while (currentBackoffMillis != BackOff.STOP) { cumulativeBackoffMillis += currentBackoffMillis; currentBackoffMillis = backOff.nextBackOffMillis(); } assertThat(cumulativeBackoffMillis, equalTo(Duration.standardMinutes(1).getMillis())); }
@Test public void testRetryFileSizeNonBatch() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class); Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(2).backoff()); when(mockStorage.objects()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000))); assertEquals( 1000, gcsUtil .getObject( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()) .getSize() .longValue()); assertEquals(BackOff.STOP, mockBackOff.nextBackOffMillis()); }
/** Tests with bounded interval, custom exponent, and unlimited retries. */ @Test public void testBoundedIntervalWithReset() throws Exception { BackOff backOff = FluentBackoff.DEFAULT .withInitialBackoff(Duration.millis(500)) .withMaxBackoff(Duration.standardSeconds(1)) .backoff(); assertThat(backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(249L), lessThan(751L))); assertThat(backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(374L), lessThan(1126L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); // Reset, should go back to short times. backOff.reset(); assertThat(backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(249L), lessThan(751L))); assertThat(backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(374L), lessThan(1126L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); assertThat( backOff.nextBackOffMillis(), allOf(greaterThanOrEqualTo(500L), lessThanOrEqualTo(1500L))); }
@Test public void testInvalidMaxRetries() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("maxRetries -1 cannot be negative"); defaultBackoff.withMaxRetries(-1); }
@Test public void testInvalidInitialBackoff() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("initialBackoff PT0S must be at least 1 millisecond"); defaultBackoff.withInitialBackoff(Duration.ZERO); }
@Test public void testInvalidCumulativeBackoff() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("maxCumulativeBackoff PT-0.002S must be at least 1 millisecond"); defaultBackoff.withMaxCumulativeBackoff(Duration.millis(-2)); }
@Test public void testInvalidMaxBackoff() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("maxBackoff PT0S must be at least 1 millisecond"); defaultBackoff.withMaxBackoff(Duration.ZERO); }
/** * Sets up external resources that are required by the example, such as Pub/Sub topics and * BigQuery tables. * * @throws IOException if there is a problem setting up the resources */ public void setup() throws IOException { Sleeper sleeper = Sleeper.DEFAULT; BackOff backOff = FluentBackoff.DEFAULT.withMaxRetries(3).withInitialBackoff(Duration.millis(200)).backoff(); Throwable lastException = null; try { do { try { setupPubsub(); setupBigQueryTable(); return; } catch (GoogleJsonResponseException e) { lastException = e; } } while (BackOffUtils.next(sleeper, backOff)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); // Ignore InterruptedException } throw new RuntimeException(lastException); }
/** Tests with bounded interval, custom exponent, limited cumulative time, and a reset. */ @Test public void testBoundedIntervalAndCumTimeWithReset() throws Exception { BackOff backOff = FluentBackoff.DEFAULT .withInitialBackoff(Duration.millis(500)) .withMaxBackoff(Duration.standardSeconds(1)) .withMaxCumulativeBackoff(Duration.standardMinutes(1)) .backoff(); assertThat(countMaximumBackoff(backOff), equalTo(Duration.standardMinutes(1).getMillis())); backOff.reset(); assertThat(countMaximumBackoff(backOff), equalTo(Duration.standardMinutes(1).getMillis())); // sanity check: should get 0 if we don't reset assertThat(countMaximumBackoff(backOff), equalTo(0L)); backOff.reset(); assertThat(countMaximumBackoff(backOff), equalTo(Duration.standardMinutes(1).getMillis())); }
@Setup public void setup() throws Exception { solrClient = spec.getConnectionConfiguration().createClient(); retryBackoff = FluentBackoff.DEFAULT .withMaxRetries(0) // default to no retrying .withInitialBackoff(RETRY_INITIAL_BACKOFF); if (spec.getRetryConfiguration() != null) { // FluentBackoff counts retries excluding the original while we count attempts // to remove ambiguity (hence the -1) retryBackoff = retryBackoff .withMaxRetries(spec.getRetryConfiguration().getMaxAttempts() - 1) .withMaxCumulativeBackoff(spec.getRetryConfiguration().getMaxDuration()); } }
/** * Discovers all shards of this file. * * <p>Because of eventual consistency, reads may discover no files or fewer files than the shard * template implies. In this case, the read is considered to have failed. */ public List<String> readFilesWithRetries() throws IOException, InterruptedException { return readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); }
@Override public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException { BackOff backoff = BackOffAdapter.toGcpBackOff( FluentBackoff.DEFAULT .withMaxRetries(maxAttempts) .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF) .withMaxBackoff(Duration.standardMinutes(1)) .backoff()); return pollJob(jobRef, Sleeper.DEFAULT, backoff); }
BackOff backoff = FluentBackoff.DEFAULT .withMaxRetries(MAX_RETRIES) .withInitialBackoff(INITIAL_BACKOFF) .backoff();
@Test public void testBackoffImplToString() throws IOException { FluentBackoff config = FluentBackoff.DEFAULT .withExponent(3.4) .withMaxRetries(4) .withInitialBackoff(Duration.standardSeconds(3)) .withMaxBackoff(Duration.standardHours(1)) .withMaxCumulativeBackoff(Duration.standardDays(1)); BackOff backOff = config.backoff(); assertEquals( "BackoffImpl{backoffConfig=" + config.toString() + "," + " currentRetry=0, currentCumulativeBackoff=PT0S}", backOff.toString()); // backoff once, ignoring result backOff.nextBackOffMillis(); // currentRetry is exact, we can test it. assertThat(backOff.toString(), containsString("currentRetry=1")); // currentCumulativeBackoff is not exact; we cannot even check that it's non-zero (randomness). } }