/** * Resets tokens to the given {@code startPosition}. This effectively causes a replay of events since that position. * <p> * Note that the new token must represent a position that is <em>before</em> the current position of the processor. * <p> * Before attempting to reset the tokens, the caller must stop this processor, as well as any instances of the * same logical processor that may be running in the cluster. Failure to do so will cause the reset to fail, * as a processor can only reset the tokens if it is able to claim them all. * * @param startPosition The token representing the position to reset the processor to. */ public void resetTokens(TrackingToken startPosition) { Assert.state(supportsReset(), () -> "The handlers assigned to this Processor do not support a reset"); Assert.state(!isRunning() && activeProcessorThreads() == 0, () -> "TrackingProcessor must be shut down before triggering a reset"); transactionManager.executeInTransaction(() -> { int[] segments = tokenStore.fetchSegments(getName()); TrackingToken[] tokens = new TrackingToken[segments.length]; for (int i = 0; i < segments.length; i++) { tokens[i] = tokenStore.fetchToken(getName(), segments[i]); } // we now have all tokens, hurray eventHandlerInvoker().performReset(); for (int i = 0; i < tokens.length; i++) { tokenStore.storeToken(ReplayToken.createReplayToken(tokens[i], startPosition), getName(), segments[i]); } }); }
long errorWaitTime = 1; try { while (state.get().isRunning() && canClaimSegment(segment.getSegmentId())) { try { eventStream = ensureEventStreamOpened(eventStream, segment); processBatch(segment, eventStream); errorWaitTime = 1; } catch (UnableToClaimTokenException e) { logger.info("Segment is owned by another node. Releasing thread to process another segment..."); releaseSegment(segment.getSegmentId()); } catch (Exception e) { releaseToken(segment); closeQuietly(eventStream); eventStream = null; doSleepFor(SECONDS.toMillis(errorWaitTime)); errorWaitTime = Math.min(errorWaitTime * 2, 60); releaseToken(segment);
/** * Resets tokens to their initial state. This effectively causes a replay. * <p> * Before attempting to reset the tokens, the caller must stop this processor, as well as any instances of the * same logical processor that may be running in the cluster. Failure to do so will cause the reset to fail, * as a processor can only reset the tokens if it is able to claim them all. */ public void resetTokens() { resetTokens(initialTrackingTokenBuilder); }
/** * Instructs the current Thread to sleep until the given deadline. This method may be overridden to check for * flags that have been set to return earlier than the given deadline. * <p> * The default implementation will sleep in blocks of 100ms, intermittently checking for the processor's state. Once * the processor stops running, this method will return immediately (after detecting the state change). * * @param millisToSleep The number of milliseconds to sleep */ protected void doSleepFor(long millisToSleep) { long deadline = System.currentTimeMillis() + millisToSleep; try { long timeLeft; while (getState().isRunning() && (timeLeft = deadline - System.currentTimeMillis()) > 0) { Thread.sleep(Math.min(timeLeft, 100)); } } catch (InterruptedException e) { logger.warn("Thread interrupted. Preparing to shut down event processor"); shutDown(); Thread.currentThread().interrupt(); } }
@Test(expected = IllegalStateException.class) public void testResetRejectedWhileRunning() { testSubject.start(); testSubject.resetTokens(); }
private void processBatch(Segment segment, BlockingStream<TrackedEventMessage<?>> eventStream) throws Exception { List<TrackedEventMessage<?>> batch = new ArrayList<>(); try { checkSegmentCaughtUp(segment, eventStream); TrackingToken lastToken = null; if (eventStream.hasNextAvailable(1, SECONDS)) { final TrackedEventMessage<?> trackedEventMessage = eventStream.nextAvailable(); lastToken = trackedEventMessage.trackingToken(); if (canHandle(trackedEventMessage, segment)) { batch.add(trackedEventMessage); } else { reportIgnored(trackedEventMessage); TrackingToken finalLastToken = lastToken; transactionManager.executeInTransaction( () -> tokenStore.storeToken(finalLastToken, getName(), segment.getSegmentId()) ); activeSegments.computeIfPresent(segment.getSegmentId(), (k, v) -> v.advancedTo(finalLastToken)); () -> tokenStore.extendClaim(getName(), segment.getSegmentId()) ); return; && eventStream.peek().filter(event -> finalLastToken.equals(event.trackingToken())).isPresent()) { final TrackedEventMessage<?> trackedEventMessage = eventStream.nextAvailable(); if (canHandle(trackedEventMessage, segment)) { batch.add(trackedEventMessage); unitOfWork.resources().put(segmentIdResourceKey, segment.getSegmentId());
@Override public PlatformInboundInstruction instruction() { Map<Integer, EventTrackerStatus> statusMap = processor.processingStatus(); .setProcessorName(processor.getName()) .setMode("Tracking") .setActiveThreads(processor.activeProcessorThreads()) .setAvailableThreads(processor.availableProcessorThreads()) .setRunning(processor.isRunning()) .setError(processor.isError()) .addAllEventTrackersInfo(trackers) .build();
@Test public void testResetBeforeStartingPerformsANormalRun() throws Exception { when(mockHandler.supportsReset()).thenReturn(true); final List<String> handled = new CopyOnWriteArrayList<>(); final List<String> handledInRedelivery = new CopyOnWriteArrayList<>(); //noinspection Duplicates doAnswer(i -> { EventMessage message = i.getArgument(0); handled.add(message.getIdentifier()); if (ReplayToken.isReplay(message)) { handledInRedelivery.add(message.getIdentifier()); } return null; }).when(mockHandler).handle(any()); testSubject.start(); testSubject.shutDown(); testSubject.resetTokens(); testSubject.start(); eventBus.publish(createEvents(4)); assertWithin(1, TimeUnit.SECONDS, () -> assertEquals(4, handled.size())); assertEquals(0, handledInRedelivery.size()); assertFalse(testSubject.processingStatus().get(0).isReplaying()); }
@Test public void testMultiThreadTokenIsStoredWhenEventIsRead() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(2); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); eventBus.publish(createEvents(2)); assertTrue("Expected Unit of Work to have reached clean up phase", countDownLatch.await(5, SECONDS)); verify(tokenStore, atLeastOnce()).storeToken(any(), any(), anyInt()); assertThat(tokenStore.fetchToken(testSubject.getName(), 0), notNullValue()); assertThat(tokenStore.fetchToken(testSubject.getName(), 1), notNullValue()); }
@Override public void run() { int waitTime = 1; String processorName = TrackingEventProcessor.this.getName(); while (getState().isRunning()) { int[] tokenStoreCurrentSegments; logger.warn("Fetch Segments for Processor '{}' failed: {}. Preparing for retry in {}s", processorName, e.getMessage(), waitTime); doSleepFor(SECONDS.toMillis(waitTime)); waitTime = Math.min(waitTime * 2, 60); + "for segment: {}. Shutting down processor [{}].", segment.getSegmentId(), getName(), return; doSleepFor(tokenClaimInterval);
@Test public void testProcessorInitializesMoreTokensThanWorkerCount() throws InterruptedException { configureProcessor(TrackingEventProcessorConfiguration.forParallelProcessing(2) .andInitialSegmentsCount(4)); testSubject.start(); // give it some time to split segments from the store and submit to executor service. Thread.sleep(200); assertThat(testSubject.activeProcessorThreads(), is(2)); int[] actual = tokenStore.fetchSegments(testSubject.getName()); Arrays.sort(actual); assertArrayEquals(new int[]{0, 1, 2, 3}, actual); }
@Test public void testTokenIsStoredOncePerEventBatch() throws Exception { testSubject = TrackingEventProcessor.builder() .name("test") .eventHandlerInvoker(eventHandlerInvoker) .messageSource(eventBus) .tokenStore(tokenStore) .transactionManager(NoTransactionManager.INSTANCE) .build(); CountDownLatch countDownLatch = new CountDownLatch(2); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); // give it a bit of time to start Thread.sleep(200); eventBus.publish(createEvents(2)); assertTrue("Expected Unit of Work to have reached clean up phase for 2 messages", countDownLatch.await(5, TimeUnit.SECONDS)); InOrder inOrder = inOrder(tokenStore); inOrder.verify(tokenStore, times(1)).extendClaim(eq(testSubject.getName()), anyInt()); inOrder.verify(tokenStore, times(1)).storeToken(any(), any(), anyInt()); assertNotNull(tokenStore.fetchToken(testSubject.getName(), 0)); }
@Test public void testReleaseSegment() { testSubject.start(); assertWithin(1, TimeUnit.SECONDS, () -> assertEquals(1, testSubject.activeProcessorThreads())); testSubject.releaseSegment(0); assertWithin(2, TimeUnit.SECONDS, () -> assertEquals(0, testSubject.activeProcessorThreads())); assertWithin(15, TimeUnit.SECONDS, () -> assertEquals(1, testSubject.activeProcessorThreads())); }
@Override public String toString() { return "TrackingSegmentWorker{" + "processor=" + getName() + ", segment=" + segment + '}'; } }
@Test public void testBlacklistingSegmentWillHaveProcessorClaimAnotherOne() { tokenStore.storeToken(new GlobalSequenceTrackingToken(1L), "test", 0); tokenStore.storeToken(new GlobalSequenceTrackingToken(2L), "test", 1); tokenStore.storeToken(new GlobalSequenceTrackingToken(2L), "test", 2); testSubject.start(); assertWithin(1, SECONDS, () -> assertEquals(0, testSubject.availableProcessorThreads())); assertEquals(new HashSet<>(asList(0, 1)), testSubject.processingStatus().keySet()); testSubject.releaseSegment(0); assertWithin(5, SECONDS, () -> assertTrue(testSubject.processingStatus().containsKey(2))); assertEquals(new HashSet<>(asList(2, 1)), testSubject.processingStatus().keySet()); assertWithin(2, SECONDS, () -> assertEquals(0, testSubject.availableProcessorThreads())); }
return null; }).when(mockHandler).handle(any()); testSubject.start(); assertEquals(2, ackedEvents.size()); testSubject.shutDown(); while (testSubject.activeProcessorThreads() > 0) { Thread.sleep(1); testSubject.start();
private BlockingStream<TrackedEventMessage<?>> ensureEventStreamOpened( BlockingStream<TrackedEventMessage<?>> eventStreamIn, Segment segment) { BlockingStream<TrackedEventMessage<?>> eventStream = eventStreamIn; if (eventStream == null && state.get().isRunning()) { final TrackingToken trackingToken = transactionManager.fetchInTransaction( () -> tokenStore.fetchToken(getName(), segment.getSegmentId()) ); logger.info("Fetched token: {} for segment: {}", trackingToken, segment); eventStream = transactionManager.fetchInTransaction( () -> doOpenStream(trackingToken)); } return eventStream; }
@Test public void testWhenFailureDuringInit() throws InterruptedException { when(tokenStore.fetchSegments(anyString())) .thenThrow(new RuntimeException("Faking issue during fetchSegments")) .thenReturn(new int[]{}) .thenReturn(new int[]{0}); doThrow(new RuntimeException("Faking issue during initializeTokenSegments")) // and on further calls .doNothing() .when(tokenStore).initializeTokenSegments(anyString(), anyInt()); testSubject.start(); Thread.sleep(2500); assertEquals(1, testSubject.activeProcessorThreads()); }
@Test public void testProcessorWorkerCount() { testSubject.start(); // give it some time to split segments from the store and submit to executor service. assertWithin(1, SECONDS, () -> assertThat(testSubject.activeProcessorThreads(), is(2))); assertThat(testSubject.processingStatus().size(), is(2)); assertTrue(testSubject.processingStatus().containsKey(0)); assertTrue(testSubject.processingStatus().containsKey(1)); assertWithin(1, SECONDS, () -> assertTrue(testSubject.processingStatus().get(0).isCaughtUp())); assertWithin(1, SECONDS, () -> assertTrue(testSubject.processingStatus().get(1).isCaughtUp())); }
@Test public void testProcessorStopsOnNonTransientExceptionWhenLoadingToken() { when(tokenStore.fetchToken("test", 0)).thenThrow(new SerializationException("Faking a serialization issue")); testSubject.start(); assertWithin( 1, TimeUnit.SECONDS, () -> assertFalse("Expected processor to have stopped", testSubject.isRunning()) ); assertWithin( 1, TimeUnit.SECONDS, () -> assertTrue("Expected processor to set the error flag", testSubject.isError()) ); assertEquals(Collections.emptyList(), sleepInstructions); }