/** * By default, we could keep the current time clock in memory, * Eventually we may need to consider the global time synchronization across all nodes * 1) When to initialize window according to start time * 2) When to close expired window according to current time * * @return StreamTimeClock instance. */ @Override public StreamTimeClock createStreamTimeClock(String streamId) { synchronized (streamIdTimeClockMap) { if (!streamIdTimeClockMap.containsKey(streamId)) { StreamTimeClock instance = new StreamTimeClockInLocalMemory(streamId); LOG.info("Created {}", instance); streamIdTimeClockMap.put(streamId, instance); } else { LOG.warn("TimeClock for stream already existss: " + streamIdTimeClockMap.get(streamId)); } return streamIdTimeClockMap.get(streamId); } }
@Test public void testSortedInPatient() throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl(); sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1h", 5000), mockCollector); List<PartitionedEvent> sortedList = new LinkedList<>(); int i = 0; while (i < 1000000) { PartitionedEvent event = MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1", System.currentTimeMillis() + i); sortHandler.nextEvent(event); sortedList.add(event); if (event.getTimestamp() > timeClock.getTime()) { timeClock.moveForward(event.getTimestamp()); } sortHandler.onTick(timeClock, System.currentTimeMillis()); i++; } sortHandler.close(); Assert.assertTrue(timeOrdering.isOrdered(sortedList)); Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get())); Assert.assertEquals(1000000, mockCollector.get().size()); }
/** * Used to debug window bucket lifecycle * <p> * Window period: PT1h, margin: 5s * * @throws InterruptedException */ @Test public void testWithSortedEventsAndExpireBySystemTime() throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl(); sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT10s", 1000), mockCollector); List<PartitionedEvent> sortedList = new LinkedList<>(); PartitionedEvent event = MockSampleMetadataFactory.createRandomSortedEventGroupedByName("sampleStream_1"); sortHandler.nextEvent(event); sortedList.add(event); timeClock.moveForward(event.getTimestamp()); sortHandler.onTick(timeClock, System.currentTimeMillis()); // Triggered to become expired by System time sortHandler.onTick(timeClock, System.currentTimeMillis() + 10 * 1000 + 1000L + 1); Assert.assertTrue(timeOrdering.isOrdered(sortedList)); Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get())); Assert.assertEquals(1, mockCollector.get().size()); sortHandler.close(); }
public void testWithSortedEvents() throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
public void testWithUnsortedEventsIn1MinuteWindow() throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl();
public void testWithUnsortedEventsIn1hWindow(int count) throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl(); sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1h", 5000), mockCollector); List<PartitionedEvent> unsortedList = new LinkedList<>(); StopWatch stopWatch = new StopWatch(); stopWatch.start(); int i = 0; while (i < count) { PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1"); sortHandler.nextEvent(event); unsortedList.add(event); if (event.getEvent().getTimestamp() > timeClock.getTime()) { timeClock.moveForward(event.getEvent().getTimestamp()); } sortHandler.onTick(timeClock, System.currentTimeMillis()); i++; } stopWatch.stop(); LOG.info("Produced {} events in {} ms", count, stopWatch.getTime()); sortHandler.close(); Assert.assertFalse(timeOrdering.isOrdered(unsortedList)); Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get())); Assert.assertTrue(mockCollector.get().size() >= 0); }
window.register(collector); StreamTimeClock clock = new StreamTimeClockInLocalMemory("sampleStream_1"); clock.moveForward(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:30,000"));