public static <T> Count<T> of(int count) { return new Count<>(count); }
@Override protected Dataset<Pair<Integer, Set<Integer>>> getOutput(Dataset<Integer> input) { return ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet())) .windowBy(Count.of(3)) .output(); }
@Override protected Dataset<Pair<Integer, Set<Integer>>> getOutput(Dataset<Integer> input) { return ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet())) .windowBy(Count.of(3)) .output(); }
@Override protected Dataset<Set<Integer>> getOutput(Dataset<Integer> input) { return ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet())) .windowBy(Count.of(3)) .outputValues(); }
public static <T> Count<T> of(int count) { return new Count<>(count); }
@Override protected Dataset<Set<Integer>> getOutput(Dataset<Integer> input) { return ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet())) .windowBy(Count.of(3)) .outputValues(); }
@Parameterized.Parameters public static List<Object[]> testParameters() { Object[][] params = { /* left-windowing, right-windowing, join-windowing, expected-failure */ {null, null, null, false}, {GlobalWindowing.get(), GlobalWindowing.get(), null, false}, {GlobalWindowing.get(), null, null, false}, {null, GlobalWindowing.get(), null, false}, {Time.of(Duration.ofMinutes(1)), null, null, true}, {null, Time.of(Duration.ofMinutes(1)), null, true}, {Time.of(Duration.ofMinutes(1)), Time.of(Duration.ofMinutes(1)), null, true}, {GlobalWindowing.get(), Time.of(Duration.ofMinutes(1)), null, true}, {Time.of(Duration.ofMinutes(1)), GlobalWindowing.get(), null, true}, {Time.of(Duration.ofMinutes(1)), null, Time.of(Duration.ofHours(1)), false}, {GlobalWindowing.get(), Time.of(Duration.ofMinutes(1)), Time.of(Duration.ofMinutes(1)), false}, {null, Time.of(Duration.ofMinutes(1)), GlobalWindowing.get(), false}, {Time.of(Duration.ofMinutes(1)), null, Count.of(10), false}, {Time.of(Duration.ofMinutes(1)), Count.of(11), GlobalWindowing.get(), false}, {Time.of(Duration.ofMinutes(1)), Count.of(11), Time.of(Duration.ofMinutes(1)), false} }; return Arrays.asList(params); }
@Parameterized.Parameters public static List<Object[]> testParameters() { Object[][] params = { /* left-windowing, right-windowing, join-windowing, expected-failure */ {null, null, null, false}, {GlobalWindowing.get(), GlobalWindowing.get(), null, false}, {GlobalWindowing.get(), null, null, false}, {null, GlobalWindowing.get(), null, false}, {Time.of(Duration.ofMinutes(1)), null, null, true}, {null, Time.of(Duration.ofMinutes(1)), null, true}, {Time.of(Duration.ofMinutes(1)), Time.of(Duration.ofMinutes(1)), null, true}, {GlobalWindowing.get(), Time.of(Duration.ofMinutes(1)), null, true}, {Time.of(Duration.ofMinutes(1)), GlobalWindowing.get(), null, true}, {Time.of(Duration.ofMinutes(1)), null, Time.of(Duration.ofHours(1)), false}, {GlobalWindowing.get(), Time.of(Duration.ofMinutes(1)), Time.of(Duration.ofMinutes(1)), false}, {null, Time.of(Duration.ofMinutes(1)), GlobalWindowing.get(), false}, {Time.of(Duration.ofMinutes(1)), null, Count.of(10), false}, {Time.of(Duration.ofMinutes(1)), Count.of(11), GlobalWindowing.get(), false}, {Time.of(Duration.ofMinutes(1)), Count.of(11), Time.of(Duration.ofMinutes(1)), false} }; return Arrays.asList(params); }
@Override protected Dataset<Pair<Integer, Long>> getOutput(Dataset<Pair<String, Integer>> input) { return ReduceStateByKey.of(input) .keyBy(e -> e.getFirst().charAt(0) - '0') .valueBy(Pair::getFirst) .stateFactory((StateFactory<String, Long, CountState<String>>) CountState::new) .mergeStatesBy(CountState::combine) // FIXME .timedBy(Pair::getSecond) and make the assertion in the validation phase stronger .windowBy(Count.of(3)) .output(); }
@Override protected Dataset<Pair<Integer, Long>> getOutput(Dataset<Pair<String, Integer>> input) { return ReduceStateByKey.of(input) .keyBy(e -> e.getFirst().charAt(0) - '0') .valueBy(Pair::getFirst) .stateFactory((StateFactory<String, Long, CountState<String>>) CountState::new) .mergeStatesBy(CountState::combine) // FIXME .timedBy(Pair::getSecond) and make the assertion in the validation phase stronger .windowBy(Count.of(3)) .output(); }
@Override protected Dataset<Pair<Integer, Integer>> getOutput(Dataset<Integer> input) { return ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .reduceBy(Fold.whileEmittingEach(0, (a, b) -> a + b)) .windowBy(Count.of(3)) .output(); }
@Override protected Dataset<Pair<Integer, Integer>> getOutput(Dataset<Integer> input) { return ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .reduceBy(Fold.whileEmittingEach(0, (a, b) -> a + b)) .windowBy(Count.of(3)) .output(); }
@Override protected Dataset<List<Pair<Integer, List<Integer>>>> getOutput(Dataset<Integer> input) { Dataset<Pair<Integer, List<Integer>>> reducedByWindow = ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toList())) .withSortedValues(Integer::compare) .windowBy(Count.of(3)) .output(); return ReduceWindow.of(reducedByWindow) .reduceBy(s -> s.collect(Collectors.toList())) .withSortedValues((l, r) -> { int cmp = l.getFirst().compareTo(r.getFirst()); if (cmp == 0) { int firstLeft = l.getSecond().get(0); int firstRight = r.getSecond().get(0); cmp = Integer.compare(firstLeft, firstRight); } return cmp; }) .windowBy(GlobalWindowing.get()) .output(); }
@Override protected Dataset<List<Pair<Integer, List<Integer>>>> getOutput(Dataset<Integer> input) { Dataset<Pair<Integer, List<Integer>>> reducedByWindow = ReduceByKey.of(input) .keyBy(e -> e % 2, Integer.class) .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toList())) .withSortedValues(Integer::compare) .windowBy(Count.of(3)) .output(); return ReduceWindow.of(reducedByWindow) .reduceBy(s -> s.collect(Collectors.toList())) .withSortedValues((l, r) -> { int cmp = l.getFirst().compareTo(r.getFirst()); if (cmp == 0) { int firstLeft = l.getSecond().get(0); int firstRight = r.getSecond().get(0); cmp = Integer.compare(firstLeft, firstRight); } return cmp; }) .windowBy(GlobalWindowing.get()) .output(); }
@Test public void testBasics() throws Exception { final Duration READ_DELAY = Duration.ofMillis(100L); ListDataSink<Set<String>> out = ListDataSink.get(); Fluent.flow("Test") .read(ListDataSource.unbounded( asList("0-one 1-two 0-three 1-four 0-five 1-six 0-seven".split(" "))) .withReadDelay(READ_DELAY)) // ~ create windows of size three .apply(input -> ReduceByKey.of(input) .keyBy(e -> "") .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet())) .windowBy(Count.of(3))) // ~ strip the needless key and flatten out the elements thereby // creating multiple elements in the output belonging to the same window .flatMap((Pair<String, Set<String>> e, Collector<String> c) -> e.getSecond().stream().forEachOrdered(c::collect)) // ~ we now expect to reconstruct the same windowing // as the very initial step .apply(input -> ReduceByKey.of(input) .keyBy(e -> "") .valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet()))) // ~ strip the needless key .mapElements(Pair::getSecond) .persist(out) .execute(new LocalExecutor()); }
.windowBy(Count.of(3)) .output();
.valueBy(e -> e) .reduceBy(s -> s.collect(Collectors.toSet())) .windowBy(Count.of(3)) .output();