private TriggerPolicy<Tuple, ?> getTriggerPolicy(Count slidingIntervalCount, Duration slidingIntervalDuration, WindowManager<Tuple> manager, EvictionPolicy<Tuple, ?> evictionPolicy) { if (slidingIntervalCount != null) { if (isTupleTs()) { return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy, manager); } else { return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy); } } else { if (isTupleTs()) { return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy, manager); } else { return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy); } } }
/** * Invokes the trigger all pending windows up to the watermark timestamp. The end ts of the window is set in the eviction policy context * so that the events falling within that window can be processed. */ private void handleWaterMarkEvent(Event<T> event) { long watermarkTs = event.getTimestamp(); long windowEndTs = nextWindowEndTs; LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs); while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); if (handler.onTrigger()) { windowEndTs += slidingIntervalMs; } else { /* * No events were found in the previous window interval. * Scan through the events in the queue to find the next * window intervals based on event ts. */ long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); LOG.debug("Next aligned window end ts {}", ts); if (ts == Long.MAX_VALUE) { LOG.debug("No events to process between {} and watermark ts {}", windowEndTs, watermarkTs); break; } windowEndTs = ts; } } nextWindowEndTs = windowEndTs; }
@Override public void track(Event<T> event) { if (started && event.isWatermark()) { handleWaterMarkEvent(event); } }
EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy);
@Override public void track(Event<T> event) { if (started && event.isWatermark()) { handleWaterMarkEvent(event); } }
long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); LOG.debug("Next aligned window end ts {}", ts); if (ts == Long.MAX_VALUE) {
@Test public void testEventTimeLag() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 632); windowManager.add(7, 629); windowManager.add(8, 636); // send a watermark event, which should trigger three windows. windowManager.add(new WaterMarkEvent<Integer>(631)); // System.out.println(listener.allOnActivationEvents); assertEquals(3, listener.allOnActivationEvents.size()); assertEquals(seq(1, 3), listener.allOnActivationEvents.get(0)); assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1)); // out of order events should be processed upto the lag assertEquals(Arrays.asList(4, 5, 7), listener.allOnActivationEvents.get(2)); }
TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy);
EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkCountEvictionPolicy<>(3); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy);
private TriggerPolicy<Tuple> getTriggerPolicy(Count slidingIntervalCount, Duration slidingIntervalDuration, WindowManager<Tuple> manager, EvictionPolicy<Tuple> evictionPolicy) { if (slidingIntervalCount != null) { if (isTupleTs()) { return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy, manager); } else { return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy); } } else { if (isTupleTs()) { return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy, manager); } else { return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy); } } }