@Override public long currentTimeMillis() { return processingTimeService.getCurrentProcessingTime(); } };
@Override public long currentProcessingTime() { return processingTimeService.getCurrentProcessingTime(); }
public void start() { timerService.registerTimer(timerService.getCurrentProcessingTime() + interval, this); }
private void scheduleNextIdleDetectionTask() { if (idleTimeout != -1) { // reset flag; if it remains true when task fires, we have detected idleness failOnNextCheck = true; nextCheck = this.timeService.registerTimer( this.timeService.getCurrentProcessingTime() + idleTimeout, new IdlenessDetectionTask()); } }
@Override public void onProcessingTime(long timestamp) throws Exception { final long currentTime = processingTimeService.getCurrentProcessingTime(); buckets.onProcessingTime(currentTime); processingTimeService.registerTimer(currentTime + bucketCheckInterval, this); }
@Override public void onProcessingTime(long timestamp) throws Exception { long currentProcessingTime = processingTimeService.getCurrentProcessingTime(); closePartFilesByTime(currentProcessingTime); processingTimeService.registerTimer(currentProcessingTime + inactiveBucketCheckInterval, this); }
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); this.processingTimeService = ((StreamingRuntimeContext) getRuntimeContext()).getProcessingTimeService(); long currentProcessingTime = processingTimeService.getCurrentProcessingTime(); processingTimeService.registerTimer(currentProcessingTime + bucketCheckInterval, this); }
bucketState.creationTime = processingTimeService.getCurrentProcessingTime();
@Override public void onProcessingTime(long timestamp) throws Exception { long minAcrossAll = Long.MAX_VALUE; boolean isEffectiveMinAggregation = false; for (KafkaTopicPartitionState<?> state : allPartitions) { // we access the current watermark for the periodic assigners under the state // lock, to prevent concurrent modification to any internal variables final long curr; //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (state) { curr = ((KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>) state).getCurrentWatermarkTimestamp(); } minAcrossAll = Math.min(minAcrossAll, curr); isEffectiveMinAggregation = true; } // emit next watermark, if there is one if (isEffectiveMinAggregation && minAcrossAll > lastWatermarkTimestamp) { lastWatermarkTimestamp = minAcrossAll; emitter.emitWatermark(new Watermark(minAcrossAll)); } // schedule the next watermark timerService.registerTimer(timerService.getCurrentProcessingTime() + interval, this); } }
@Override protected void processAndCollect(T element) { lastRecordTime = this.timeService.getCurrentProcessingTime(); output.collect(reuse.replace(element, lastRecordTime)); // this is to avoid lock contention in the lockingObject by // sending the watermark before the firing of the watermark // emission task. if (lastRecordTime > nextWatermarkTime) { // in case we jumped some watermarks, recompute the next watermark time final long watermarkTime = lastRecordTime - (lastRecordTime % watermarkInterval); nextWatermarkTime = watermarkTime + watermarkInterval; output.emitWatermark(new Watermark(watermarkTime)); // we do not need to register another timer here // because the emitting task will do so. } }
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); state = new State<>(); processingTimeService = ((StreamingRuntimeContext) getRuntimeContext()).getProcessingTimeService(); long currentProcessingTime = processingTimeService.getCurrentProcessingTime(); processingTimeService.registerTimer(currentProcessingTime + inactiveBucketCheckInterval, this); this.clock = new Clock() { @Override public long currentTimeMillis() { return processingTimeService.getCurrentProcessingTime(); } }; }
@Override public void invoke(T value) throws Exception { Path bucketPath = bucketer.getBucketPath(clock, new Path(basePath), value); long currentProcessingTime = processingTimeService.getCurrentProcessingTime(); BucketState<T> bucketState = state.getBucketState(bucketPath); if (bucketState == null) { bucketState = new BucketState<>(currentProcessingTime); state.addBucketState(bucketPath, bucketState); } if (shouldRoll(bucketState, currentProcessingTime)) { openNewPartFile(bucketPath, bucketState); } bucketState.writer.write(value); bucketState.lastWrittenToTime = currentProcessingTime; }
@Override public void open() throws Exception { super.open(); currentWatermark = Long.MIN_VALUE; watermarkInterval = getExecutionConfig().getAutoWatermarkInterval(); if (watermarkInterval > 0) { long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); } }
@Override public void onProcessingTime(long timestamp) throws Exception { // register next timer long newWatermark = userFunction.getCurrentWatermark(); if (newWatermark > currentWatermark) { currentWatermark = newWatermark; // emit watermark output.emitWatermark(new Watermark(currentWatermark)); } long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); }
@Override public void open() throws Exception { super.open(); watermarkInterval = getExecutionConfig().getAutoWatermarkInterval(); if (watermarkInterval > 0) { long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); } currentWatermark = Long.MIN_VALUE; }
private AutomaticWatermarkContext( final Output<StreamRecord<T>> output, final long watermarkInterval, final ProcessingTimeService timeService, final Object checkpointLock, final StreamStatusMaintainer streamStatusMaintainer, final long idleTimeout) { super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout); this.output = Preconditions.checkNotNull(output, "The output cannot be null."); Preconditions.checkArgument(watermarkInterval >= 1L, "The watermark interval cannot be smaller than 1 ms."); this.watermarkInterval = watermarkInterval; this.reuse = new StreamRecord<>(null); this.lastRecordTime = Long.MIN_VALUE; long now = this.timeService.getCurrentProcessingTime(); this.nextWatermarkTimer = this.timeService.registerTimer(now + watermarkInterval, new WatermarkEmittingTask(this.timeService, checkpointLock, output)); }
@Override public void onProcessingTime(long timestamp) throws Exception { // register next timer Watermark newWatermark = userFunction.getCurrentWatermark(); if (newWatermark != null && newWatermark.getTimestamp() > currentWatermark) { currentWatermark = newWatermark.getTimestamp(); // emit watermark output.emitWatermark(newWatermark); } long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); }
@Override public void processElement(StreamRecord<IN> element) throws Exception { final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); if (timeout > 0L) { // register a timeout for this AsyncStreamRecordBufferEntry long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer( timeoutTimestamp, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { userFunction.timeout(element.getValue(), streamRecordBufferEntry); } }); // Cancel the timer once we've completed the stream record buffer entry. This will remove // the register trigger task streamRecordBufferEntry.onComplete( (StreamElementQueueEntry<Collection<OUT>> value) -> { timerFuture.cancel(true); }, executor); } addAsyncBufferEntry(streamRecordBufferEntry); userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); }
long timestamp = getProcessingTimeService().getCurrentProcessingTime(); advanceTime(nfaState, timestamp); processEvent(nfaState, element.getValue(), timestamp);
assertEquals(Long.MIN_VALUE, testHarness.getProcessingTimeService().getCurrentProcessingTime()); assertEquals(testHarness.getProcessingTimeService().getCurrentProcessingTime(), 11); assertEquals(testHarness.getProcessingTimeService().getCurrentProcessingTime(), 16);