@Override protected Dataset<Triple<TimeInterval, Integer, String>> getOutput(Dataset<Pair<String, Integer>> input) { input = AssignEventTime.of(input).using(e -> e.getSecond()).output(); Dataset<Pair<Integer, String>> reduced = ReduceStateByKey.of(input) .keyBy(e -> e.getFirst().charAt(0) - '0') .valueBy(e -> e.getFirst().substring(2)) .stateFactory((StateFactory<String, String, AccState<String>>) AccState::new) .mergeStatesBy(AccState::combine) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output(); return FlatMap.of(reduced) .using((UnaryFunctor<Pair<Integer, String>, Triple<TimeInterval, Integer, String>>) (elem, context) -> context.collect(Triple.of((TimeInterval) context.getWindow(), elem.getFirst(), elem.getSecond()))) .output(); }
@Override protected Dataset<Triple<TimeInterval, Integer, String>> getOutput(Dataset<Pair<String, Integer>> input) { input = AssignEventTime.of(input).using(e -> e.getSecond()).output(); Dataset<Pair<Integer, String>> reduced = ReduceStateByKey.of(input) .keyBy(e -> e.getFirst().charAt(0) - '0') .valueBy(e -> e.getFirst().substring(2)) .stateFactory((StateFactory<String, String, AccState<String>>) AccState::new) .mergeStatesBy(AccState::combine) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output(); return FlatMap.of(reduced) .using((UnaryFunctor<Pair<Integer, String>, Triple<TimeInterval, Integer, String>>) (elem, context) -> context.collect(Triple.of((TimeInterval) context.getWindow(), elem.getFirst(), elem.getSecond()))) .output(); }
.valueBy(e -> 1) .combineBy(Sums.ofInts()) .windowBy(TimeSliding.of(params.getLongStats(), params.getShortStats())) .output();
@Test public void testTimeSlidingLabelAssignment() { TimeSliding<Long> windowing = TimeSliding .of(Duration.ofHours(1), Duration.ofMinutes(20)); UnaryFunction<Long, Long> eventTimeAssigner = e -> e * 1000L; long[] data = { 3590, 3600, 3610, 3800, 7190, 7200, 7210 }; for (long event : data) { Iterable<TimeInterval> labels = windowing .assignWindowsToElement(new Elem<>( GlobalWindowing.Window.get(), event, eventTimeAssigner.apply(event))); // verify window count assertEquals(3, Iterables.size(labels)); // verify that each window contains the original event for (TimeInterval l : labels) { long stamp = event * 1000L; assertTrue(stamp >= l.getStartMillis()); assertTrue(stamp <= l.getEndMillis()); } } }
.valueBy(e -> 1L) .reduceBy(Sums.ofLongs()) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output();
.valueBy(e -> 1L) .combineBy(Sums.ofLongs()) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output();
@Test public void testWindowAssignment() { TimeSliding<?> windowing = TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5));