/** * Construct a {@link Timer} with given window size * * @param name name of this timer * @param windowMs the window size. unit is millisecond * @param clock the clock for the reservoir */ public Timer(String name, long windowMs, Clock clock) { this(name, new SlidingTimeWindowReservoir(windowMs, clock)); }
@Override public void update(long value) { if (count.incrementAndGet() % REMOVE_IN_UPDATE_THRESHOLD == 0) { removeExpireValues(); } storage.put(getUpdatingTime(), value); }
@Test public void testUpdateSizeSnapshot() { SlidingTimeWindowReservoir slidingTimeWindowReservoir = new SlidingTimeWindowReservoir(300, 8, clock); when(clock.currentTimeMillis()).thenReturn(0L); slidingTimeWindowReservoir.update(1L); when(clock.currentTimeMillis()).thenReturn(1L); slidingTimeWindowReservoir.update(2L); when(clock.currentTimeMillis()).thenReturn(2L); slidingTimeWindowReservoir.update(3L); assertEquals(3, slidingTimeWindowReservoir.size()); Snapshot snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 2L, 3L))); assertEquals(3, snapshot.getSize()); }
@Test public void testRemoveExpiredValues() { SlidingTimeWindowReservoir slidingTimeWindowReservoir = new SlidingTimeWindowReservoir(300, 8, clock); when(clock.currentTimeMillis()).thenReturn(0L); slidingTimeWindowReservoir.update(1L); when(clock.currentTimeMillis()).thenReturn(100L); slidingTimeWindowReservoir.update(2L); when(clock.currentTimeMillis()).thenReturn(301L); slidingTimeWindowReservoir.update(3L); when(clock.currentTimeMillis()).thenReturn(500L); slidingTimeWindowReservoir.update(4L); Snapshot snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(3L, 4L))); assertEquals(2, snapshot.getSize()); }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { Throwable handlerError = eventHubNonTransientError.get(); /* * We will retry for non transient error by instantiating a new EventHubs client if * 1. Last retry happened more than CONFIG_MIN_RETRY_INTERVAL_MS ms ago. Otherwise we ignore * 2. We haven't reached CONFIG_MAX_RETRY_COUNT allowed within the CONFIG_RETRY_WINDOW_MS window. * Otherwise we throw */ if (handlerError != null && clock.currentTimeMillis() - lastRetryTs > config.getMinRetryIntervalMs(systemName)) { int currentRetryCount = recentRetryAttempts.size(); long maxRetryCount = config.getMaxRetryCount(systemName); if (currentRetryCount < maxRetryCount) { LOG.warn("Received non transient error. Will retry.", handlerError); LOG.info("Current retry count within window: {}. max retry count allowed: {}. window size: {} ms", currentRetryCount, maxRetryCount, config.getRetryWindowMs(systemName)); long now = clock.currentTimeMillis(); recentRetryAttempts.update(now); lastRetryTs = now; reconnectTaskStatus = reconnectTaskRunner.submit(this::renewEventHubsClient); } else { LOG.error("Retries exhausted. Reached max allowed retries: ({}) within window {} ms", currentRetryCount, config.getRetryWindowMs(systemName)); String msg = "Received a non transient error from event hub partition receiver"; throw new SamzaException(msg, handlerError); } } return super.poll(systemStreamPartitions, timeout); }
@Override public int size() { removeExpireValues(); return storage.size(); }
Future lastReconnectTask = consumer.reconnectTaskStatus; lastReconnectTask.get(10000, TimeUnit.MILLISECONDS); // should return instantaneously Assert.assertEquals(consumer.recentRetryAttempts.size(), 1); Assert.assertEquals(consumer.recentRetryAttempts.size(), 0); eventHubClientWrapperFactory.triggerError(consumer.streamPartitionHandlers, new EventHubException(false /* is transient */, "test")); lastReconnectTask = consumer.reconnectTaskStatus; lastReconnectTask.get(10000, TimeUnit.MILLISECONDS); // should return instantaneously Assert.assertEquals(consumer.recentRetryAttempts.size(), 1);
/** * Remove the values that are earlier than current window */ private void removeExpireValues() { storage.headMap(getUpdatingTime() - windowMs).clear(); }
@Test public void testDuplicateTime() { SlidingTimeWindowReservoir slidingTimeWindowReservoir = new SlidingTimeWindowReservoir(300, 2, clock); when(clock.currentTimeMillis()).thenReturn(1L); slidingTimeWindowReservoir.update(1L); slidingTimeWindowReservoir.update(2L); Snapshot snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 2L))); assertEquals(2, snapshot.getSize()); // update causes collision, will override the last update slidingTimeWindowReservoir.update(3L); snapshot = slidingTimeWindowReservoir.getSnapshot(); assertTrue(snapshot.getValues().containsAll(Arrays.asList(1L, 3L))); assertEquals(2, snapshot.getSize()); }
@Override public int size() { removeExpireValues(); return storage.size(); }
/** * Remove the values that are earlier than current window */ private void removeExpireValues() { storage.headMap(getUpdatingTime() - windowMs).clear(); }
/** * Construct a {@link Timer} with given window size * * @param name name of this timer * @param windowMs the window size. unit is millisecond * @param clock the clock for the reservoir */ public Timer(String name, long windowMs, Clock clock) { this(name, new SlidingTimeWindowReservoir(windowMs, clock)); }
@Override public void update(long value) { if (count.incrementAndGet() % REMOVE_IN_UPDATE_THRESHOLD == 0) { removeExpireValues(); } storage.put(getUpdatingTime(), value); }
@Override public Snapshot getSnapshot() { removeExpireValues(); return new Snapshot(storage.values()); } }
/** * Default constructor. It uses {@link SlidingTimeWindowReservoir} as the * default reservoir. * * @param name name of this timer */ public Timer(String name) { this(name, new SlidingTimeWindowReservoir()); }
@Override public Snapshot getSnapshot() { removeExpireValues(); return new Snapshot(storage.values()); } }
/** * Construct a {@link Timer} with given window size and collision buffer * * @param name name of this timer * @param windowMs the window size. unit is millisecond * @param collisionBuffer amount of collisions allowed in one millisecond. * @param clock the clock for the reservoir */ public Timer(String name, long windowMs, int collisionBuffer, Clock clock) { this(name, new SlidingTimeWindowReservoir(windowMs, collisionBuffer, clock)); }
/** * Default constructor. It uses {@link SlidingTimeWindowReservoir} as the * default reservoir. * * @param name name of this timer */ public Timer(String name) { this(name, new SlidingTimeWindowReservoir()); }
/** * Construct a {@link Timer} with given window size and collision buffer * * @param name name of this timer * @param windowMs the window size. unit is millisecond * @param collisionBuffer amount of collisions allowed in one millisecond. * @param clock the clock for the reservoir */ public Timer(String name, long windowMs, int collisionBuffer, Clock clock) { this(name, new SlidingTimeWindowReservoir(windowMs, collisionBuffer, clock)); }
prefetchCount = config.getPrefetchCount(systemName); recentRetryAttempts = new SlidingTimeWindowReservoir(config.getRetryWindowMs(systemName), clock);