@Override public Iterable<OUT> call(Iterable<IN> input) { if (collector == null) { collector = new FunctionCollectorMem<>(accumulatorProvider); } collector.clear(); reducer.apply(StreamSupport.stream(input.spliterator(), false), collector); return () -> collector.getOutputIterator(); } }
@Override public Iterator<SparkElement<WID, OUT>> call(SparkElement<WID, IN> elem) { final WID window = elem.getWindow(); final long timestamp = getTimestamp(elem); FunctionCollectorMem<OUT> collector = getContext(); // setup user collector collector.clear(); collector.setWindow(window); functor.apply(elem.getElement(), collector); // wrap output in WindowedElement return Iterators.transform(collector.getOutputIterator(), e -> new SparkElement<>(window, timestamp, e)); }
FunctionCollectorMem<OUT> getCollector() { if (cachedCollector == null) { cachedCollector = new FunctionCollectorMem<>(accumulators); } return cachedCollector; }
first == null ? window.maxTimestamp() - 1 : first.getTimestamp(), second == null ? window.maxTimestamp() - 1 : second.getTimestamp()); collector.clear(); collector.setWindow(window); operator .getJoiner() collector); return Iterators.transform( collector.getOutputIterator(), e -> new SparkElement<>(window, maxTimestamp, Pair.of(t._1.key(), e))); },
private FunctionCollectorMem<OUT> getContext() { if (cachedCollector == null) { cachedCollector = new FunctionCollectorMem<>(accumulators); } return cachedCollector; } }
first == null ? 0L : first.getTimestamp(), second == null ? 0L : second.getTimestamp()); collector.clear(); collector.setWindow(GlobalWindowing.Window.get()); operator .getJoiner() collector); return Iterators.transform( collector.getOutputIterator(), e -> new SparkElement<>(