/** * Windows this {@code KeyedStream} into sliding time windows. * * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time * characteristic set using * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)} * * @param size The size of the window. */ public WindowedStream<T, KEY, TimeWindow> timeWindow(Time size, Time slide) { if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) { return window(SlidingProcessingTimeWindows.of(size, slide)); } else { return window(SlidingEventTimeWindows.of(size, slide)); } }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
@Test public void testWindowAssignment() { WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingProcessingTimeWindows assigner = SlidingProcessingTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000)); when(mockContext.getCurrentProcessingTime()).thenReturn(0L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(-4000, 1000), timeWindow(-3000, 2000), timeWindow(-2000, 3000), timeWindow(-1000, 4000), timeWindow(0, 5000))); when(mockContext.getCurrentProcessingTime()).thenReturn(4999L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(0, 5000), timeWindow(1000, 6000), timeWindow(2000, 7000), timeWindow(3000, 8000), timeWindow(4000, 9000))); when(mockContext.getCurrentProcessingTime()).thenReturn(5000L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(1000, 6000), timeWindow(2000, 7000), timeWindow(3000, 8000), timeWindow(4000, 9000), timeWindow(5000, 10000))); }
@Test public void testProperties() { SlidingProcessingTimeWindows assigner = SlidingProcessingTimeWindows.of(Time.seconds(5), Time.milliseconds(100)); assertFalse(assigner.isEventTime()); assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig())); assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(ProcessingTimeTrigger.class)); } }
private long getPaneSize() { Preconditions.checkArgument( legacyWindowOperatorType == LegacyWindowOperatorType.FAST_ACCUMULATING || legacyWindowOperatorType == LegacyWindowOperatorType.FAST_AGGREGATING); final long paneSlide; if (windowAssigner instanceof SlidingProcessingTimeWindows) { SlidingProcessingTimeWindows timeWindows = (SlidingProcessingTimeWindows) windowAssigner; paneSlide = ArithmeticUtils.gcd(timeWindows.getSize(), timeWindows.getSlide()); } else { TumblingProcessingTimeWindows timeWindows = (TumblingProcessingTimeWindows) windowAssigner; paneSlide = timeWindows.getSize(); // this is valid as windowLength == windowSlide == timeWindows.getSize } return paneSlide; }
/** * Windows this {@code DataStream} into sliding time windows. * * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic * set using * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)} * * <p>Note: This operation is inherently non-parallel since all elements have to pass through * the same operator instance. * * @param size The size of the window. */ public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size, Time slide) { if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) { return windowAll(SlidingProcessingTimeWindows.of(size, slide)); } else { return windowAll(SlidingEventTimeWindows.of(size, slide)); } }
@Test public void testTimeUnits() { // sanity check with one other time unit WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingProcessingTimeWindows assigner = SlidingProcessingTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500)); when(mockContext.getCurrentProcessingTime()).thenReturn(100L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(-4500, 500), timeWindow(-3500, 1500), timeWindow(-2500, 2500), timeWindow(-1500, 3500), timeWindow(-500, 4500))); when(mockContext.getCurrentProcessingTime()).thenReturn(5499L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(500, 5500), timeWindow(1500, 6500), timeWindow(2500, 7500), timeWindow(3500, 8500), timeWindow(4500, 9500))); when(mockContext.getCurrentProcessingTime()).thenReturn(5100L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(500, 5500), timeWindow(1500, 6500), timeWindow(2500, 7500), timeWindow(3500, 8500), timeWindow(4500, 9500))); }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to time windows based on the element timestamp and offset. * * <p>For example, if you want window a stream by hour,but window begins at the 15th minutes * of each hour, you can use {@code of(Time.hours(1),Time.minutes(15))},then you will get * time windows start at 0:15:00,1:15:00,2:15:00,etc. * * <p>Rather than that,if you are living in somewhere which is not using UTC±00:00 time, * such as China which is using UTC+08:00,and you want a time window with size of one day, * and window begins at every 00:00:00 of local time,you may use {@code of(Time.days(1),Time.hours(-8))}. * The parameter of offset is {@code Time.hours(-8))} since UTC+08:00 is 8 hours earlier than UTC time. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @param offset The offset which window start would be shifted by. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide, Time offset) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), offset.toMilliseconds()); }
@Test public void testInvalidParameters() { try { SlidingProcessingTimeWindows.of(Time.seconds(-2), Time.seconds(1)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingProcessingTimeWindows.of(Time.seconds(2), Time.seconds(-1)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingProcessingTimeWindows.of(Time.seconds(-20), Time.seconds(10), Time.seconds(-1)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingProcessingTimeWindows.of(Time.seconds(20), Time.seconds(10), Time.seconds(-11)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingProcessingTimeWindows.of(Time.seconds(20), Time.seconds(10), Time.seconds(11)); fail("should fail"); } catch (IllegalArgumentException e) {
@Test public void testWindowAssignmentWithOffset() { WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingProcessingTimeWindows assigner = SlidingProcessingTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100)); when(mockContext.getCurrentProcessingTime()).thenReturn(100L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(-3900, 1100), timeWindow(-2900, 2100), timeWindow(-1900, 3100), timeWindow(-900, 4100), timeWindow(100, 5100))); when(mockContext.getCurrentProcessingTime()).thenReturn(5099L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(100, 5100), timeWindow(1100, 6100), timeWindow(2100, 7100), timeWindow(3100, 8100), timeWindow(4100, 9100))); when(mockContext.getCurrentProcessingTime()).thenReturn(5100L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(1100, 6100), timeWindow(2100, 7100), timeWindow(3100, 8100), timeWindow(4100, 9100), timeWindow(5100, 10100))); }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
@Test @SuppressWarnings("rawtypes") public void testReduceProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .reduce(new DummyReducer()); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
@Test public void testWindowAssignmentWithNegativeOffset() { WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingProcessingTimeWindows assigner = SlidingProcessingTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100)); when(mockContext.getCurrentProcessingTime()).thenReturn(0L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(-4100, 900), timeWindow(-3100, 1900), timeWindow(-2100, 2900), timeWindow(-1100, 3900), timeWindow(-100, 4900))); when(mockContext.getCurrentProcessingTime()).thenReturn(4899L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(-100, 4900), timeWindow(900, 5900), timeWindow(1900, 6900), timeWindow(2900, 7900), timeWindow(3900, 8900))); when(mockContext.getCurrentProcessingTime()).thenReturn(4900L); assertThat(assigner.assignWindows("String", Long.MIN_VALUE, mockContext), containsInAnyOrder( timeWindow(900, 5900), timeWindow(1900, 6900), timeWindow(2900, 7900), timeWindow(3900, 8900), timeWindow(4900, 9900))); }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
@Test @SuppressWarnings("rawtypes") public void testReduceProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .keyBy(new TupleKeySelector()) .window(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .reduce(new DummyReducer()); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
@Test public void testAggregateProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .aggregate(new DummyAggregationFunction()); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to time windows based on the element timestamp and offset. * * <p>For example, if you want window a stream by hour,but window begins at the 15th minutes * of each hour, you can use {@code of(Time.hours(1),Time.minutes(15))},then you will get * time windows start at 0:15:00,1:15:00,2:15:00,etc. * * <p>Rather than that,if you are living in somewhere which is not using UTC±00:00 time, * such as China which is using UTC+08:00,and you want a time window with size of one day, * and window begins at every 00:00:00 of local time,you may use {@code of(Time.days(1),Time.hours(-8))}. * The parameter of offset is {@code Time.hours(-8))} since UTC+08:00 is 8 hours earlier than UTC time. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @param offset The offset which window start would be shifted by. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide, Time offset) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), offset.toMilliseconds() % slide.toMilliseconds()); }
@Test @SuppressWarnings("rawtypes") public void testFoldProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple3<String, String, Integer>> window = source .windowAll(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .fold(new Tuple3<>("", "", 0), new DummyFolder()); OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
/** * Creates a new {@code SlidingProcessingTimeWindows} {@link WindowAssigner} that assigns * elements to time windows based on the element timestamp and offset. * * <p>For example, if you want window a stream by hour,but window begins at the 15th minutes * of each hour, you can use {@code of(Time.hours(1),Time.minutes(15))},then you will get * time windows start at 0:15:00,1:15:00,2:15:00,etc. * * <p>Rather than that,if you are living in somewhere which is not using UTC±00:00 time, * such as China which is using UTC+08:00,and you want a time window with size of one day, * and window begins at every 00:00:00 of local time,you may use {@code of(Time.days(1),Time.hours(-8))}. * The parameter of offset is {@code Time.hours(-8))} since UTC+08:00 is 8 hours earlier than UTC time. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @param offset The offset which window start would be shifted by. * @return The time policy. */ public static SlidingProcessingTimeWindows of(Time size, Time slide, Time offset) { return new SlidingProcessingTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), offset.toMilliseconds() % slide.toMilliseconds()); }