/** * Windows this {@code KeyedStream} into sliding time windows. * * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time * characteristic set using * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)} * * @param size The size of the window. */ public WindowedStream<T, KEY, TimeWindow> timeWindow(Time size, Time slide) { if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) { return window(SlidingProcessingTimeWindows.of(size, slide)); } else { return window(SlidingEventTimeWindows.of(size, slide)); } }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
@Test public void testWindowAssignment() { WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000)); assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder( timeWindow(-4000, 1000), timeWindow(-3000, 2000), timeWindow(-2000, 3000), timeWindow(-1000, 4000), timeWindow(0, 5000))); assertThat(assigner.assignWindows("String", 4999L, mockContext), containsInAnyOrder( timeWindow(0, 5000), timeWindow(1000, 6000), timeWindow(2000, 7000), timeWindow(3000, 8000), timeWindow(4000, 9000))); assertThat(assigner.assignWindows("String", 5000L, mockContext), containsInAnyOrder( timeWindow(1000, 6000), timeWindow(2000, 7000), timeWindow(3000, 8000), timeWindow(4000, 9000), timeWindow(5000, 10000))); }
@Test public void testProperties() { SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.milliseconds(100)); assertTrue(assigner.isEventTime()); assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig())); assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class)); } }
/** * Windows this {@code DataStream} into sliding time windows. * * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic * set using * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)} * * <p>Note: This operation is inherently non-parallel since all elements have to pass through * the same operator instance. * * @param size The size of the window. */ public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size, Time slide) { if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) { return windowAll(SlidingProcessingTimeWindows.of(size, slide)); } else { return windowAll(SlidingEventTimeWindows.of(size, slide)); } }
@Test public void testWindowAssignmentWithOffset() { WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(100)); assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder( timeWindow(-3900, 1100), timeWindow(-2900, 2100), timeWindow(-1900, 3100), timeWindow(-900, 4100), timeWindow(100, 5100))); assertThat(assigner.assignWindows("String", 5099L, mockContext), containsInAnyOrder( timeWindow(100, 5100), timeWindow(1100, 6100), timeWindow(2100, 7100), timeWindow(3100, 8100), timeWindow(4100, 9100))); assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder( timeWindow(1100, 6100), timeWindow(2100, 7100), timeWindow(3100, 8100), timeWindow(4100, 9100), timeWindow(5100, 10100))); }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to time windows based on the element timestamp and offset. * * <p>For example, if you want window a stream by hour,but window begins at the 15th minutes * of each hour, you can use {@code of(Time.hours(1),Time.minutes(15))},then you will get * time windows start at 0:15:00,1:15:00,2:15:00,etc. * * <p>Rather than that,if you are living in somewhere which is not using UTC±00:00 time, * such as China which is using UTC+08:00,and you want a time window with size of one day, * and window begins at every 00:00:00 of local time,you may use {@code of(Time.days(1),Time.hours(-8))}. * The parameter of offset is {@code Time.hours(-8))} since UTC+08:00 is 8 hours earlier than UTC time. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @param offset The offset which window start would be shifted by. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide, Time offset) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), offset.toMilliseconds()); }
static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) { long slideSize = pt.getLong( TEST_SLIDE_SIZE.key(), TEST_SLIDE_SIZE.defaultValue()); long slideFactor = pt.getInt( TEST_SLIDE_FACTOR.key(), TEST_SLIDE_FACTOR.defaultValue() ); return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize)); }
@Test public void testWindowAssignmentWithNegativeOffset() { WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.milliseconds(5000), Time.milliseconds(1000), Time.milliseconds(-100)); assertThat(assigner.assignWindows("String", 0L, mockContext), containsInAnyOrder( timeWindow(-4100, 900), timeWindow(-3100, 1900), timeWindow(-2100, 2900), timeWindow(-1100, 3900), timeWindow(-100, 4900))); assertThat(assigner.assignWindows("String", 4899L, mockContext), containsInAnyOrder( timeWindow(-100, 4900), timeWindow(900, 5900), timeWindow(1900, 6900), timeWindow(2900, 7900), timeWindow(3900, 8900))); assertThat(assigner.assignWindows("String", 4900L, mockContext), containsInAnyOrder( timeWindow(900, 5900), timeWindow(1900, 6900), timeWindow(2900, 7900), timeWindow(3900, 8900), timeWindow(4900, 9900))); }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
@Test public void testInvalidParameters() { try { SlidingEventTimeWindows.of(Time.seconds(-2), Time.seconds(1)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingEventTimeWindows.of(Time.seconds(2), Time.seconds(-1)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingEventTimeWindows.of(Time.seconds(-20), Time.seconds(10), Time.seconds(-1)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingEventTimeWindows.of(Time.seconds(20), Time.seconds(10), Time.seconds(-11)); fail("should fail"); } catch (IllegalArgumentException e) { SlidingEventTimeWindows.of(Time.seconds(20), Time.seconds(10), Time.seconds(11)); fail("should fail"); } catch (IllegalArgumentException e) {
@Test public void testTimeUnits() { // sanity check with one other time unit WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); SlidingEventTimeWindows assigner = SlidingEventTimeWindows.of(Time.seconds(5), Time.seconds(1), Time.milliseconds(500)); assertThat(assigner.assignWindows("String", 100L, mockContext), containsInAnyOrder( timeWindow(-4500, 500), timeWindow(-3500, 1500), timeWindow(-2500, 2500), timeWindow(-1500, 3500), timeWindow(-500, 4500))); assertThat(assigner.assignWindows("String", 5499L, mockContext), containsInAnyOrder( timeWindow(500, 5500), timeWindow(1500, 6500), timeWindow(2500, 7500), timeWindow(3500, 8500), timeWindow(4500, 9500))); assertThat(assigner.assignWindows("String", 5100L, mockContext), containsInAnyOrder( timeWindow(500, 5500), timeWindow(1500, 6500), timeWindow(2500, 7500), timeWindow(3500, 8500), timeWindow(4500, 9500))); }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
/** * .aggregate() does not support RichAggregateFunction, since the AggregateFunction is used internally * in an {@code AggregatingState}. */ @Test(expected = UnsupportedOperationException.class) public void testAggregateWithRichFunctionFails() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); source .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .aggregate(new DummyRichAggregationFunction<Tuple2<String, Integer>>()); fail("exception was not thrown"); }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to sliding time windows based on the element timestamp. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), 0); }
/** * .aggregate() does not support RichAggregateFunction, since the AggregationFunction is used internally * in a {@code AggregatingState}. */ @Test(expected = UnsupportedOperationException.class) public void testAggregateWithRichFunctionFails() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); source .keyBy(0) .window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .aggregate(new DummyRichAggregationFunction<Tuple2<String, Integer>>()); fail("exception was not thrown"); }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to time windows based on the element timestamp and offset. * * <p>For example, if you want window a stream by hour,but window begins at the 15th minutes * of each hour, you can use {@code of(Time.hours(1),Time.minutes(15))},then you will get * time windows start at 0:15:00,1:15:00,2:15:00,etc. * * <p>Rather than that,if you are living in somewhere which is not using UTC±00:00 time, * such as China which is using UTC+08:00,and you want a time window with size of one day, * and window begins at every 00:00:00 of local time,you may use {@code of(Time.days(1),Time.hours(-8))}. * The parameter of offset is {@code Time.hours(-8))} since UTC+08:00 is 8 hours earlier than UTC time. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @param offset The offset which window start would be shifted by. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide, Time offset) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), offset.toMilliseconds() % slide.toMilliseconds()); }
/** * .reduce() does not support RichReduceFunction, since the reduce function is used internally * in a {@code ReducingState}. */ @Test(expected = UnsupportedOperationException.class) public void testReduceWithRichReducerFails() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); source .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .reduce(new RichReduceFunction<Tuple2<String, Integer>>() { private static final long serialVersionUID = -6448847205314995812L; @Override public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception { return null; } }); fail("exception was not thrown"); }
/** * Creates a new {@code SlidingEventTimeWindows} {@link WindowAssigner} that assigns * elements to time windows based on the element timestamp and offset. * * <p>For example, if you want window a stream by hour,but window begins at the 15th minutes * of each hour, you can use {@code of(Time.hours(1),Time.minutes(15))},then you will get * time windows start at 0:15:00,1:15:00,2:15:00,etc. * * <p>Rather than that,if you are living in somewhere which is not using UTC±00:00 time, * such as China which is using UTC+08:00,and you want a time window with size of one day, * and window begins at every 00:00:00 of local time,you may use {@code of(Time.days(1),Time.hours(-8))}. * The parameter of offset is {@code Time.hours(-8))} since UTC+08:00 is 8 hours earlier than UTC time. * * @param size The size of the generated windows. * @param slide The slide interval of the generated windows. * @param offset The offset which window start would be shifted by. * @return The time policy. */ public static SlidingEventTimeWindows of(Time size, Time slide, Time offset) { return new SlidingEventTimeWindows(size.toMilliseconds(), slide.toMilliseconds(), offset.toMilliseconds() % slide.toMilliseconds()); }
/** * .fold() does not support RichFoldFunction, since the fold function is used internally * in a {@code FoldingState}. */ @Test(expected = UnsupportedOperationException.class) public void testFoldWithRichFolderFails() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); source .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .fold(new Tuple2<>("", 0), new RichFoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() { private static final long serialVersionUID = -6448847205314995812L; @Override public Tuple2<String, Integer> fold(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception { return null; } }); fail("exception was not thrown"); }