@Test public void testFirstTimeSuccessPut() throws Exception { String tableId = "testFirstTimeSuccessPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(writeFn).putAsync(anyString(), anyString()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); retryIO.putAsync("foo", "bar").get(); verify(writeFn, times(1)).putAsync(anyString(), anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testFirstTimeSuccessGet() throws Exception { String tableId = "testFirstTimeSuccessGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals("bar", retryIO.getAsync("foo").get()); verify(readFn, times(1)).getAsync(anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testRetryExhaustedAttemptsGet() throws Exception { String tableId = "testRetryExhaustedAttempts"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(readFn, times(11)).getAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedAttemptsPut() throws Exception { String tableId = "testRetryExhaustedAttemptsPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).deleteAllAsync(any()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.deleteAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(writeFn, times(11)).deleteAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedTimePut() throws Exception { String tableId = "testRetryExhaustedTimePut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).deleteAsync(anyString()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.deleteAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(writeFn, atLeast(3)).deleteAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedTimeGet() throws Exception { String tableId = "testRetryExhaustedTime"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(readFn, atLeast(3)).getAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testDeleteAll() throws Exception { ReadWriteTable table = createTable(false); table.deleteAll(Collections.emptyList()); table.deleteAllAsync(Collections.emptyList()).get(); verify(kvStore, times(2)).deleteAll(any()); Assert.assertEquals(2, numDeleteAlls.getCount()); Assert.assertTrue(deleteAllNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, putAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPuts.getCount()); Assert.assertEquals(0, numPutAlls.getCount()); Assert.assertEquals(0, numDeletes.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testGetAll() throws Exception { ReadWriteTable table = createTable(false); Assert.assertEquals(values, table.getAll(keys)); Assert.assertEquals(values, table.getAllAsync(keys).get()); verify(kvStore, times(2)).getAll(any()); Assert.assertEquals(Collections.emptyMap(), table.getAll(Collections.emptyList())); Assert.assertEquals(2, numMissedLookups.getCount()); Assert.assertEquals(3, numGetAlls.getCount()); Assert.assertTrue(getAllNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, numGets.getCount()); Assert.assertEquals(0, getNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, getCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testPutWithRetryDisabled() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(false).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).putAsync(any(), any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAsync("foo", "bar").get(); fail(); } catch (ExecutionException e) { } verify(writeFn, times(1)).putAsync(any(), any()); assertEquals(0, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount()); assertEquals(0, table.writeRetryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testPut() throws Exception { ReadWriteTable table = createTable(false); table.put("k1", "v1"); table.putAsync("k2", "v2").get(); table.putAsync("k3", null).get(); verify(kvStore, times(2)).put(any(), any()); verify(kvStore, times(1)).delete(any()); Assert.assertEquals(2, numPuts.getCount()); Assert.assertEquals(1, numDeletes.getCount()); Assert.assertTrue(putNs.getSnapshot().getAverage() > 0); Assert.assertTrue(deleteNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPutAlls.getCount()); Assert.assertEquals(0, numDeleteAlls.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testDelete() throws Exception { ReadWriteTable table = createTable(false); table.delete(""); table.deleteAsync("").get(); verify(kvStore, times(2)).delete(any()); Assert.assertEquals(2, numDeletes.getCount()); Assert.assertTrue(deleteNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, putAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPuts.getCount()); Assert.assertEquals(0, numPutAlls.getCount()); Assert.assertEquals(0, numDeleteAlls.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testPutAll() throws Exception { ReadWriteTable table = createTable(false); List<Entry> entries = Arrays.asList(new Entry("k1", "v1"), new Entry("k2", null)); table.putAll(entries); table.putAllAsync(entries).get(); verify(kvStore, times(2)).putAll(any()); verify(kvStore, times(2)).deleteAll(any()); Assert.assertEquals(2, numPutAlls.getCount()); Assert.assertEquals(2, numDeleteAlls.getCount()); Assert.assertTrue(putAllNs.getSnapshot().getAverage() > 0); Assert.assertTrue(deleteAllNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPuts.getCount()); Assert.assertEquals(0, numDeletes.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testDefaultTimerUpdateAndGetSnapshot() { Timer timer = new Timer("test", 300, clock); timer.update(1L); timer.update(2L); Snapshot snapshot = timer.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 2L))); assertEquals(2, snapshot.getValues().size()); }
@Test public void testPutWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).putAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAllAsync(Arrays.asList(new Entry(1, 2))).get(); fail(); } catch (ExecutionException e) { } verify(writeFn, atLeast(11)).putAllAsync(any()); assertEquals(10, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(1, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testFlush() { ReadWriteTable table = createTable(false); table.flush(); table.flush(); // Note: store.flush() is NOT called here verify(kvStore, times(0)).flush(); Assert.assertEquals(2, numFlushes.getCount()); Assert.assertTrue(flushNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, putAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPuts.getCount()); Assert.assertEquals(0, numPutAlls.getCount()); Assert.assertEquals(0, numDeletes.getCount()); Assert.assertEquals(0, numDeleteAlls.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testGetWithPermFailureOnTimeout() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(3)).getAsync(any()); assertTrue(table.readRetryMetrics.retryCount.getCount() >= 3); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGet() throws Exception { ReadWriteTable table = createTable(false); Assert.assertEquals("v1", table.get("k1")); Assert.assertEquals("v2", table.getAsync("k2").get()); Assert.assertNull(table.get("k3")); verify(kvStore, times(3)).get(any()); Assert.assertEquals(3, numGets.getCount()); Assert.assertEquals(1, numMissedLookups.getCount()); Assert.assertTrue(getNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, numGetAlls.getCount()); Assert.assertEquals(0, getAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, getCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testTimerDisabled() throws Exception { ReadWriteTable table = createTable(true); table.get(""); table.getAsync("").get(); table.getAll(keys); table.getAllAsync(keys).get(); Assert.assertEquals(2, numGets.getCount()); Assert.assertEquals(4, numMissedLookups.getCount()); Assert.assertEquals(2, numGetAlls.getCount()); Assert.assertEquals(0, getNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, getAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, getCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testGetWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(11)).getAsync(any()); assertEquals(10, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testTimerWithDifferentWindowSize() { Timer timer = new Timer("test", 300, clock); timer.update(1L); timer.update(2L); timer.update(3L); Snapshot snapshot = timer.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 2L, 3L))); assertEquals(3, snapshot.getValues().size()); // The time is 500 for update(4L) because getSnapshot calls clock once + 3 // updates that call clock 3 times timer.update(4L); Snapshot snapshot2 = timer.getSnapshot(); assertTrue(snapshot2.getValues().containsAll(Arrays.asList(3L, 4L))); assertEquals(2, snapshot2.getValues().size()); } }