@Override public CompletableFuture<Map<K, V>> getAllAsync(List<K> keys) { return readFn.getAllAsync(keys); }
/** * Fetch the table {@code records} for specified {@code keys}. This method must be thread-safe. * The default implementation calls getAllAsync and blocks on the completion afterwards. * @param keys keys for the table records * @return all records for the specified keys. */ default Map<K, V> getAll(Collection<K> keys) { try { return getAllAsync(keys).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET_ALL failed for " + keys, e); } }
/** * Fetch the table {@code records} for specified {@code keys}. This method must be thread-safe. * The default implementation calls getAllAsync and blocks on the completion afterwards. * @param keys keys for the table records * @return all records for the specified keys. */ default Map<K, V> getAll(Collection<K> keys) { try { return getAllAsync(keys).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET_ALL failed for " + keys, e); } }
/** * Fetch the table {@code records} for specified {@code keys}. This method must be thread-safe. * The default implementation calls getAllAsync and blocks on the completion afterwards. * @param keys keys for the table records * @return all records for the specified keys. */ default Map<K, V> getAll(Collection<K> keys) { try { return getAllAsync(keys).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET_ALL failed for " + keys, e); } }
/** * Fetch the table {@code records} for specified {@code keys}. This method must be thread-safe. * The default implementation calls getAllAsync and blocks on the completion afterwards. * @param keys keys for the table records * @return all records for the specified keys. */ default Map<K, V> getAll(Collection<K> keys) { try { return getAllAsync(keys).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET_ALL failed for " + keys, e); } }
/** * Fetch the table {@code records} for specified {@code keys}. This method must be thread-safe. * The default implementation calls getAllAsync and blocks on the completion afterwards. * @param keys keys for the table records * @return all records for the specified keys. */ default Map<K, V> getAll(Collection<K> keys) { try { return getAllAsync(keys).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET_ALL failed for " + keys, e); } }
@Override public CompletableFuture<Map<K, V>> getAllAsync(Collection<K> keys) { return failsafe(retryPolicy, retryMetrics, retryExecutor) .future(() -> readFn.getAllAsync(keys)) .exceptionally(e -> { throw new SamzaException("Failed to get the records for " + keys + " after retries.", e); }); }
@Override public CompletableFuture<Map<K, V>> getAllAsync(Collection<K> keys) { return failsafe(retryPolicy, retryMetrics, retryExecutor) .future(() -> readFn.getAllAsync(keys)) .exceptionally(e -> { throw new SamzaException("Failed to get the records for " + keys + " after retries.", e); }); }
@Override public CompletableFuture<Map<K, V>> getAllAsync(Collection<K> keys) { return failsafe(retryPolicy, retryMetrics, retryExecutor) .future(() -> readFn.getAllAsync(keys)) .exceptionally(e -> { throw new SamzaException("Failed to get the records for " + keys + " after retries.", e); }); }
@Override public CompletableFuture<Map<K, V>> getAllAsync(Collection<K> keys) { return failsafe(retryPolicy, retryMetrics, retryExecutor) .future(() -> readFn.getAllAsync(keys)) .exceptionally(e -> { throw new SamzaException("Failed to get the records for " + keys + " after retries.", e); }); }
@Override public CompletableFuture<Map<K, V>> getAllAsync(Collection<K> keys) { return failsafe(retryPolicy, retryMetrics, retryExecutor) .future(() -> readFn.getAllAsync(keys)) .exceptionally(e -> { throw new SamzaException("Failed to get the records for " + keys + " after retries.", e); }); }
@Test public void testGetAllAsync() { int times = 0; roTable.getAllAsync(Arrays.asList(1, 2)); verify(readFn, times(++times)).getAllAsync(any()); rwTable.getAllAsync(Arrays.asList(1, 2)); verify(readFn, times(++times)).getAllAsync(any()); }
private void doTestGetAll(boolean sync, boolean error, boolean partial) throws Exception { TableReadFunction<String, String> readFn = mock(TableReadFunction.class); Map<String, String> res = new HashMap<>(); res.put("foo1", "bar1"); if (!partial) { res.put("foo2", "bar2"); } CompletableFuture<Map<String, String>> future; if (error) { future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("Test exception")); } else { future = CompletableFuture.completedFuture(res); } // Sync is backed by async so needs to mock the async method doReturn(future).when(readFn).getAllAsync(any()); RemoteTable<String, String> table = getTable("testGetAll-" + sync + error + partial, readFn, null, false); Assert.assertEquals(res, sync ? table.getAll(Arrays.asList("foo1", "foo2")) : table.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(table.readRateLimiter, times(1)).throttle(anyCollection()); }
@Test public void testGetThrottling() throws Exception { TableRateLimiter readRateLimiter = mock(TableRateLimiter.class); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Map<String, String> result = new HashMap<>(); result.put("foo", "bar"); doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRateLimitedTable table = new AsyncRateLimitedTable("t1", delegate, readRateLimiter, null, schedExec); table.init(TestRemoteTable.getMockContext()); Assert.assertEquals("bar", table.getAsync("foo").get()); verify(readFn, times(1)).getAsync(any()); verify(readRateLimiter, times(1)).throttle(anyString()); verify(readRateLimiter, times(0)).throttle(anyList()); Assert.assertEquals(result, table.getAllAsync(Arrays.asList("")).get()); verify(readFn, times(1)).getAllAsync(any()); verify(readRateLimiter, times(1)).throttle(anyList()); verify(readRateLimiter, times(1)).throttle(anyString()); }
@Test public void testRetryEngagedGet() throws Exception { String tableId = "testRetryEngagedGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); int [] times = new int[] {0}; Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times[0] > 0) { future.complete(map); } else { times[0]++; future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals(map, retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedAttemptsGet() throws Exception { String tableId = "testRetryExhaustedAttempts"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(readFn, times(11)).getAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetAllWithOneRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); AtomicInteger times = new AtomicInteger(); Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times.get() > 0) { future.complete(map); } else { times.incrementAndGet(); future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); assertEquals(map, table.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); assertEquals(1, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetWithoutRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Map<String, String> result = new HashMap<>(); result.put("foo", "bar"); doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); int times = 0; table.init(TestRemoteTable.getMockContext()); verify(readFn, times(1)).init(any()); assertEquals("bar", table.getAsync("foo").get()); verify(readFn, times(1)).getAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(result, table.getAllAsync(Arrays.asList("foo")).get()); verify(readFn, times(1)).getAllAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertNull(table.writeRetryMetrics); table.close(); verify(readFn, times(1)).close(); }
records.put("foo1", "bar1"); records.put("foo2", "bar2"); doReturn(CompletableFuture.completedFuture(records)).when(readFn).getAllAsync(any()); Assert.assertEquals(cachingTable.getAllAsync(Arrays.asList("foo1", "foo2")).get(), records); doReturn(CompletableFuture.completedFuture(Collections.singletonMap("foo3", "bar3"))).when(readFn).getAllAsync(any()); records = cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo3")).get(); Assert.assertEquals(records.get("foo3"), "bar3"); doReturn(exFuture).when(readFn).getAllAsync(any()); cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo3")).get();
@Test public void testGetWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(11)).getAsync(any()); assertEquals(10, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }