/** Configures the Publisher's retry parameters. */ public Builder setRetrySettings(RetrySettings retrySettings) { Preconditions.checkArgument( retrySettings.getTotalTimeout().compareTo(MIN_TOTAL_TIMEOUT) >= 0); Preconditions.checkArgument( retrySettings.getInitialRpcTimeout().compareTo(MIN_RPC_TIMEOUT) >= 0); this.retrySettings = retrySettings; return this; }
private void verifyRetrySettings(Set<Code> retryCodes, RetrySettings retrySettings) { assertThat(retryCodes).containsAllOf(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); assertThat(retrySettings.getTotalTimeout()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getInitialRetryDelay()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getRetryDelayMultiplier()).isAtLeast(1.0); assertThat(retrySettings.getMaxRetryDelay()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getInitialRpcTimeout()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getRpcTimeoutMultiplier()).isAtLeast(1.0); assertThat(retrySettings.getMaxRpcTimeout()).isGreaterThan(Duration.ZERO); } }
private void verifyRetrySettingAreSane(Set<Code> retryCodes, RetrySettings retrySettings) { assertThat(retryCodes).containsAllOf(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); assertThat(retrySettings.getTotalTimeout()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getInitialRetryDelay()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getRetryDelayMultiplier()).isAtLeast(1.0); assertThat(retrySettings.getMaxRetryDelay()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getInitialRpcTimeout()).isGreaterThan(Duration.ZERO); assertThat(retrySettings.getRpcTimeoutMultiplier()).isAtLeast(1.0); assertThat(retrySettings.getMaxRpcTimeout()).isGreaterThan(Duration.ZERO); } }
@Test public void testReadRowsSettings() { ServerStreamingCallSettings.Builder<ReadRowsRequest, ReadRowsResponse> builder = EnhancedBigQueryStorageStubSettings.newBuilder().readRowsSettings(); assertThat(builder.getRetryableCodes()).containsAllOf(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); RetrySettings retrySettings = builder.getRetrySettings(); assertThat(retrySettings.getInitialRetryDelay()).isEqualTo(Duration.ofMillis(100L)); assertThat(retrySettings.getRetryDelayMultiplier()).isWithin(1e-6).of(1.3); assertThat(retrySettings.getMaxRetryDelay()).isEqualTo(Duration.ofMinutes(1L)); assertThat(retrySettings.getInitialRpcTimeout()).isEqualTo(Duration.ofDays(1L)); assertThat(retrySettings.getRpcTimeoutMultiplier()).isWithin(1e-6).of(1.0); assertThat(retrySettings.getMaxRpcTimeout()).isEqualTo(Duration.ofDays(1L)); assertThat(retrySettings.getTotalTimeout()).isEqualTo(Duration.ofDays(1L)); assertThat(builder.getIdleTimeout()).isEqualTo(Duration.ZERO); }
/** * Creates a callable chain to handle point ReadRows RPCs. The chain will: * * <ul> * <li>Convert a {@link Query} into a {@link com.google.bigtable.v2.ReadRowsRequest} and * dispatch the RPC. * <li>Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row * implementation can be configured in by the {@code rowAdapter} parameter. * <li>Retry/resume on failure. * <li>Filter out marker rows. * </ul> */ public <RowT> UnaryCallable<Query, RowT> createReadRowCallable(RowAdapter<RowT> rowAdapter) { return createReadRowsCallable( ServerStreamingCallSettings.<Query, Row>newBuilder() .setRetryableCodes(settings.readRowSettings().getRetryableCodes()) .setRetrySettings(settings.readRowSettings().getRetrySettings()) .setIdleTimeout(settings.readRowSettings().getRetrySettings().getTotalTimeout()) .build(), rowAdapter) .first(); }
private AwaitReplicationCallable createAwaitReplicationCallable() { // TODO(igorbernstein2): expose polling settings RetrySettings pollingSettings = RetrySettings.newBuilder() // use overall timeout from checkConsistencyCallable // NOTE: The overall timeout might exceed this value due to underlying retries .setTotalTimeout( settings.checkConsistencySettings().getRetrySettings().getTotalTimeout()) // Use constant polling with jitter .setInitialRetryDelay(Duration.ofSeconds(10)) .setRetryDelayMultiplier(1.0) .setMaxRetryDelay(Duration.ofSeconds(10)) .setJittered(true) // These rpc timeouts are ignored, instead the rpc timeouts defined for // generateConsistencyToken and checkConsistency callables will be used. .setInitialRpcTimeout(Duration.ZERO) .setMaxRpcTimeout(Duration.ZERO) .setRpcTimeoutMultiplier(1.0) .build(); return AwaitReplicationCallable.create( generateConsistencyTokenCallable(), checkConsistencyCallable(), clientContext, pollingSettings); }
@Test public void testCreateFromStream() { RemoteComputeHelper helper = RemoteComputeHelper.create(PROJECT_ID, JSON_KEY_STREAM); ComputeOptions options = helper.getOptions(); assertEquals(PROJECT_ID, options.getProjectId()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getConnectTimeout()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getReadTimeout()); assertEquals(10, options.getRetrySettings().getMaxAttempts()); assertEquals(Duration.ofMillis(30000), options.getRetrySettings().getMaxRetryDelay()); assertEquals(Duration.ofMillis(120000), options.getRetrySettings().getTotalTimeout()); assertEquals(Duration.ofMillis(250), options.getRetrySettings().getInitialRetryDelay()); }
@Test public void testCreateFromStream() { RemoteStorageHelper helper = RemoteStorageHelper.create(PROJECT_ID, JSON_KEY_STREAM); StorageOptions options = helper.getOptions(); assertEquals(PROJECT_ID, options.getProjectId()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getConnectTimeout()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getReadTimeout()); assertEquals(10, options.getRetrySettings().getMaxAttempts()); assertEquals(Duration.ofMillis(30000), options.getRetrySettings().getMaxRetryDelay()); assertEquals(Duration.ofMillis(120000), options.getRetrySettings().getTotalTimeout()); assertEquals(Duration.ofMillis(250), options.getRetrySettings().getInitialRetryDelay()); } }
@Test public void testCreateFromStream() { RemoteBigQueryHelper helper = RemoteBigQueryHelper.create(PROJECT_ID, JSON_KEY_STREAM); BigQueryOptions options = helper.getOptions(); assertEquals(PROJECT_ID, options.getProjectId()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getConnectTimeout()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getReadTimeout()); assertEquals(10, options.getRetrySettings().getMaxAttempts()); assertEquals(Duration.ofMillis(30000), options.getRetrySettings().getMaxRetryDelay()); assertEquals(Duration.ofMillis(120000), options.getRetrySettings().getTotalTimeout()); assertEquals(Duration.ofMillis(250), options.getRetrySettings().getInitialRetryDelay()); } }
.sampleRowKeysSettings() .setSimpleTimeoutNoRetries( settings.sampleRowKeysSettings().getRetrySettings().getTotalTimeout()) .setRetryableCodes(settings.sampleRowKeysSettings().getRetryableCodes());
private RequestConfig getRequestConfig() { RequestConfig.Builder configBuilder = RequestConfig.custom(); if (photosLibrarySettings.uploadMediaItemSettings().getRetrySettings().getTotalTimeout() != UNLIMITED_TIMEOUT) { configBuilder.setConnectionRequestTimeout( Math.toIntExact( photosLibrarySettings .uploadMediaItemSettings() .getRetrySettings() .getTotalTimeout() .get(MILLIS))); } return configBuilder.build(); }
private void checkForTimeout(long initialMillis) throws TimeoutException { if (photosLibrarySettings.uploadMediaItemSettings().getRetrySettings().getTotalTimeout() != UNLIMITED_TIMEOUT) { long duration = clientContext.getClock().millisTime() - initialMillis; if (duration > photosLibrarySettings .uploadMediaItemSettings() .getRetrySettings() .getTotalTimeout() .get(MILLIS)) { throw new TimeoutException(ExceptionStrings.UPLOAD_TIMED_OUT); } } }
public RetrySettings build() { RetrySettings params = autoBuild(); if (params.getTotalTimeout().toMillis() < 0) { throw new IllegalStateException("total timeout must not be negative"); } if (params.getInitialRetryDelay().toMillis() < 0) { throw new IllegalStateException("initial retry delay must not be negative"); } if (params.getRetryDelayMultiplier() < 1.0) { throw new IllegalStateException("retry delay multiplier must be at least 1"); } if (params.getMaxRetryDelay().compareTo(params.getInitialRetryDelay()) < 0) { throw new IllegalStateException("max retry delay must not be shorter than initial delay"); } if (params.getMaxAttempts() < 0) { throw new IllegalStateException("max attempts must be non-negative"); } if (params.getInitialRpcTimeout().toMillis() < 0) { throw new IllegalStateException("initial rpc timeout must not be negative"); } if (params.getMaxRpcTimeout().compareTo(params.getInitialRpcTimeout()) < 0) { throw new IllegalStateException("max rpc timeout must not be shorter than initial timeout"); } if (params.getRpcTimeoutMultiplier() < 1.0) { throw new IllegalStateException("rpc timeout multiplier must be at least 1"); } return params; }
@Test public void testSetSimpleTimeoutNoRetries() { UnaryCallSettings.Builder builder = new UnaryCallSettings.Builder(); builder.setSimpleTimeoutNoRetries(Duration.ofSeconds(13)); Truth.assertThat(builder.getRetryableCodes().size()).isEqualTo(0); Truth.assertThat(builder.getRetrySettings().getMaxAttempts()).isEqualTo(1); Truth.assertThat(builder.getRetrySettings().getTotalTimeout()) .isEqualTo(Duration.ofSeconds(13)); } }
/** * Returns {@code true} if another attempt should be made, or {@code false} otherwise. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if {@code nextAttemptSettings} does not exceed either maxAttempts limit or * totalTimeout limit, or {@code false} otherwise */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) { RetrySettings globalSettings = nextAttemptSettings.getGlobalSettings(); long totalTimeSpentNanos = clock.nanoTime() - nextAttemptSettings.getFirstAttemptStartTimeNanos() + nextAttemptSettings.getRandomizedRetryDelay().toNanos(); return totalTimeSpentNanos <= globalSettings.getTotalTimeout().toNanos() && (globalSettings.getMaxAttempts() <= 0 || nextAttemptSettings.getAttemptCount() < globalSettings.getMaxAttempts()); }
/** * Returns {@code true} if another attempt should be made, or {@code false} otherwise. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if {@code nextAttemptSettings} does not exceed either maxAttempts limit or * totalTimeout limit, or {@code false} otherwise */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) { RetrySettings globalSettings = nextAttemptSettings.getGlobalSettings(); long totalTimeSpentNanos = clock.nanoTime() - nextAttemptSettings.getFirstAttemptStartTimeNanos() + nextAttemptSettings.getRandomizedRetryDelay().toNanos(); return totalTimeSpentNanos <= globalSettings.getTotalTimeout().toNanos() && (globalSettings.getMaxAttempts() <= 0 || nextAttemptSettings.getAttemptCount() < globalSettings.getMaxAttempts()); }
protected Long getNextBackoff() { if (currentBackoff == null) { // Historically, the client waited for "total timeout" after the first failure. For now, // that behavior is preserved, even though that's not the ideal. // // TODO: Think through retries, and create policy that works with the mental model most // users would have of relating to retries. That would likely involve updating some // default settings in addition to changing the algorithm. currentBackoff = exponentialRetryAlgorithm.createFirstAttempt(); } currentBackoff = exponentialRetryAlgorithm.createNextAttempt(currentBackoff); if (!exponentialRetryAlgorithm.shouldRetry(currentBackoff)) { // TODO: consider creating a subclass of exponentialRetryAlgorithm to encapsulate this logic long timeLeftNs = currentBackoff.getGlobalSettings().getTotalTimeout().toNanos() - (clock.nanoTime() - currentBackoff.getFirstAttemptStartTimeNanos()); long timeLeftMs = TimeUnit.NANOSECONDS.toMillis(timeLeftNs); if (timeLeftMs > currentBackoff.getGlobalSettings().getInitialRetryDelay().toMillis()) { // The backoff algorithm doesn't always wait until the timeout is achieved. Wait // one final time so that retries hit return timeLeftMs; } else { // Finish for real. return null; } } else { return currentBackoff.getRetryDelay().toMillis(); } }
/** * Creates a callable chain to handle point ReadRows RPCs. The chain will: * * <ul> * <li>Convert a {@link Query} into a {@link com.google.bigtable.v2.ReadRowsRequest} and * dispatch the RPC. * <li>Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row * implementation can be configured in by the {@code rowAdapter} parameter. * <li>Retry/resume on failure. * <li>Filter out marker rows. * </ul> */ public <RowT> UnaryCallable<Query, RowT> createReadRowCallable(RowAdapter<RowT> rowAdapter) { return createReadRowsCallable( ServerStreamingCallSettings.<Query, Row>newBuilder() .setRetryableCodes(settings.readRowSettings().getRetryableCodes()) .setRetrySettings(settings.readRowSettings().getRetrySettings()) .setIdleTimeout(settings.readRowSettings().getRetrySettings().getTotalTimeout()) .build(), rowAdapter) .first(); }
private AwaitReplicationCallable createAwaitReplicationCallable() { // TODO(igorbernstein2): expose polling settings RetrySettings pollingSettings = RetrySettings.newBuilder() // use overall timeout from checkConsistencyCallable // NOTE: The overall timeout might exceed this value due to underlying retries .setTotalTimeout( settings.checkConsistencySettings().getRetrySettings().getTotalTimeout()) // Use constant polling with jitter .setInitialRetryDelay(Duration.ofSeconds(10)) .setRetryDelayMultiplier(1.0) .setMaxRetryDelay(Duration.ofSeconds(10)) .setJittered(true) // These rpc timeouts are ignored, instead the rpc timeouts defined for // generateConsistencyToken and checkConsistency callables will be used. .setInitialRpcTimeout(Duration.ZERO) .setMaxRpcTimeout(Duration.ZERO) .setRpcTimeoutMultiplier(1.0) .build(); return AwaitReplicationCallable.create( generateConsistencyTokenCallable(), checkConsistencyCallable(), clientContext, pollingSettings); }
@Test public void retrySettingsMerge() { RetrySettings.Builder builder = RetrySettings.newBuilder() .setTotalTimeout(Duration.ofMillis(45000)) .setInitialRpcTimeout(Duration.ofMillis(2000)) .setRpcTimeoutMultiplier(1.5) .setMaxRpcTimeout(Duration.ofMillis(30000)) .setInitialRetryDelay(Duration.ofMillis(100)) .setRetryDelayMultiplier(1.2) .setMaxRetryDelay(Duration.ofMillis(1000)); RetrySettings.Builder mergedBuilder = RetrySettings.newBuilder(); mergedBuilder.merge(builder); RetrySettings settingsA = builder.build(); RetrySettings settingsB = mergedBuilder.build(); Truth.assertThat(settingsA.getTotalTimeout()).isEqualTo(settingsB.getTotalTimeout()); Truth.assertThat(settingsA.getInitialRetryDelay()).isEqualTo(settingsB.getInitialRetryDelay()); Truth.assertThat(settingsA.getRpcTimeoutMultiplier()) .isWithin(0) .of(settingsB.getRpcTimeoutMultiplier()); Truth.assertThat(settingsA.getMaxRpcTimeout()).isEqualTo(settingsB.getMaxRpcTimeout()); Truth.assertThat(settingsA.getInitialRetryDelay()).isEqualTo(settingsB.getInitialRetryDelay()); Truth.assertThat(settingsA.getRetryDelayMultiplier()) .isWithin(0) .of(settingsB.getRetryDelayMultiplier()); Truth.assertThat(settingsA.getMaxRetryDelay()).isEqualTo(settingsB.getMaxRetryDelay()); }