@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { Throwable handlerError = eventHubNonTransientError.get(); /* * We will retry for non transient error by instantiating a new EventHubs client if * 1. Last retry happened more than CONFIG_MIN_RETRY_INTERVAL_MS ms ago. Otherwise we ignore * 2. We haven't reached CONFIG_MAX_RETRY_COUNT allowed within the CONFIG_RETRY_WINDOW_MS window. * Otherwise we throw */ if (handlerError != null && clock.currentTimeMillis() - lastRetryTs > config.getMinRetryIntervalMs(systemName)) { int currentRetryCount = recentRetryAttempts.size(); long maxRetryCount = config.getMaxRetryCount(systemName); if (currentRetryCount < maxRetryCount) { LOG.warn("Received non transient error. Will retry.", handlerError); LOG.info("Current retry count within window: {}. max retry count allowed: {}. window size: {} ms", currentRetryCount, maxRetryCount, config.getRetryWindowMs(systemName)); long now = clock.currentTimeMillis(); recentRetryAttempts.update(now); lastRetryTs = now; reconnectTaskStatus = reconnectTaskRunner.submit(this::renewEventHubsClient); } else { LOG.error("Retries exhausted. Reached max allowed retries: ({}) within window {} ms", currentRetryCount, config.getRetryWindowMs(systemName)); String msg = "Received a non transient error from event hub partition receiver"; throw new SamzaException(msg, handlerError); } } return super.poll(systemStreamPartitions, timeout); }
@Test public void testRemoveExpiredValues() { SlidingTimeWindowReservoir slidingTimeWindowReservoir = new SlidingTimeWindowReservoir(300, 8, clock); when(clock.currentTimeMillis()).thenReturn(0L); slidingTimeWindowReservoir.update(1L); when(clock.currentTimeMillis()).thenReturn(100L); slidingTimeWindowReservoir.update(2L); when(clock.currentTimeMillis()).thenReturn(301L); slidingTimeWindowReservoir.update(3L); when(clock.currentTimeMillis()).thenReturn(500L); slidingTimeWindowReservoir.update(4L); Snapshot snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(3L, 4L))); assertEquals(2, snapshot.getSize()); }
@Test public void testUpdateSizeSnapshot() { SlidingTimeWindowReservoir slidingTimeWindowReservoir = new SlidingTimeWindowReservoir(300, 8, clock); when(clock.currentTimeMillis()).thenReturn(0L); slidingTimeWindowReservoir.update(1L); when(clock.currentTimeMillis()).thenReturn(1L); slidingTimeWindowReservoir.update(2L); when(clock.currentTimeMillis()).thenReturn(2L); slidingTimeWindowReservoir.update(3L); assertEquals(3, slidingTimeWindowReservoir.size()); Snapshot snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 2L, 3L))); assertEquals(3, snapshot.getSize()); }
@Test public void testDuplicateTime() { SlidingTimeWindowReservoir slidingTimeWindowReservoir = new SlidingTimeWindowReservoir(300, 2, clock); when(clock.currentTimeMillis()).thenReturn(1L); slidingTimeWindowReservoir.update(1L); slidingTimeWindowReservoir.update(2L); Snapshot snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 2L))); assertEquals(2, snapshot.getSize()); // update causes collision, will override the last update slidingTimeWindowReservoir.update(3L); snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 3L))); assertEquals(2, snapshot.getSize()); }