@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
public RetriableReadFunction(TableRetryPolicy policy, TableReadFunction<K, V> readFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(readFn); Preconditions.checkNotNull(retryExecutor); this.readFn = readFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableReadFunction(TableRetryPolicy policy, TableReadFunction<K, V> readFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(readFn); Preconditions.checkNotNull(retryExecutor); this.readFn = readFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableReadFunction(TableRetryPolicy policy, TableReadFunction<K, V> readFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(readFn); Preconditions.checkNotNull(retryExecutor); this.readFn = readFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableReadFunction(TableRetryPolicy policy, TableReadFunction<K, V> readFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(readFn); Preconditions.checkNotNull(retryExecutor); this.readFn = readFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableReadFunction(TableRetryPolicy policy, TableReadFunction<K, V> readFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(readFn); Preconditions.checkNotNull(retryExecutor); this.readFn = readFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public AsyncRetriableTable(String tableId, AsyncReadWriteTable<K, V> table, TableRetryPolicy readRetryPolicy, TableRetryPolicy writeRetryPolicy, ScheduledExecutorService retryExecutor, TableReadFunction readFn, TableWriteFunction writeFn) { Preconditions.checkNotNull(tableId, "null tableId"); Preconditions.checkNotNull(table, "null table"); Preconditions.checkNotNull(retryExecutor, "null retryExecutor"); Preconditions.checkArgument(readRetryPolicy != null || writeRetryPolicy != null, "both readRetryPolicy and writeRetryPolicy are null"); this.tableId = tableId; this.table = table; this.retryExecutor = retryExecutor; if (readRetryPolicy != null && readFn != null) { Predicate<Throwable> readRetryPredicate = readRetryPolicy.getRetryPredicate(); readRetryPolicy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || readRetryPredicate.test(ex)); this.readRetryPolicy = FailsafeAdapter.valueOf(readRetryPolicy); } else { this.readRetryPolicy = null; } if (writeRetryPolicy != null && writeFn != null) { Predicate<Throwable> writeRetryPredicate = writeRetryPolicy.getRetryPredicate(); writeRetryPolicy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || writeRetryPredicate.test(ex)); this.writeRetryPolicy = FailsafeAdapter.valueOf(writeRetryPolicy); } else { this.writeRetryPolicy = null; } }
private void doTestGet(boolean sync, boolean error, boolean retry) throws Exception { String tableId = "testGet-" + sync + error + retry; TableReadFunction<String, String> readFn = mock(TableReadFunction.class); // Sync is backed by async so needs to mock the async method CompletableFuture<String> future; if (error) { future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("Test exception")); if (!retry) { doReturn(future).when(readFn).getAsync(anyString()); } else { final int [] times = new int[] {0}; doAnswer(args -> times[0]++ == 0 ? future : CompletableFuture.completedFuture("bar")) .when(readFn).getAsync(anyString()); } } else { future = CompletableFuture.completedFuture("bar"); doReturn(future).when(readFn).getAsync(anyString()); } if (retry) { doReturn(true).when(readFn).isRetriable(any()); } RemoteTable<String, String> table = getTable(tableId, readFn, null, retry); Assert.assertEquals("bar", sync ? table.get("foo") : table.getAsync("foo").get()); verify(table.readRateLimiter, times(error && retry ? 2 : 1)).throttle(anyString()); }
@Test public void testFirstTimeSuccessGet() throws Exception { String tableId = "testFirstTimeSuccessGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals("bar", retryIO.getAsync("foo").get()); verify(readFn, times(1)).getAsync(anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testRetryEngagedGet() throws Exception { String tableId = "testRetryEngagedGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); int [] times = new int[] {0}; Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times[0] > 0) { future.complete(map); } else { times[0]++; future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals(map, retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedAttemptsGet() throws Exception { String tableId = "testRetryExhaustedAttempts"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(readFn, times(11)).getAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedTimeGet() throws Exception { String tableId = "testRetryExhaustedTime"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(readFn, atLeast(3)).getAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetWithoutRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Map<String, String> result = new HashMap<>(); result.put("foo", "bar"); doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); int times = 0; table.init(TestRemoteTable.getMockContext()); verify(readFn, times(1)).init(any()); assertEquals("bar", table.getAsync("foo").get()); verify(readFn, times(1)).getAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(result, table.getAllAsync(Arrays.asList("foo")).get()); verify(readFn, times(1)).getAllAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertNull(table.writeRetryMetrics); table.close(); verify(readFn, times(1)).close(); }
@Test public void testGetWithRetryDisabled() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(false).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, times(1)).getAsync(any()); assertEquals(0, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testGetWithPermFailureOnTimeout() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(3)).getAsync(any()); assertTrue(table.readRetryMetrics.retryCount.getCount() >= 3); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(11)).getAsync(any()); assertEquals(10, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }