@Override @SuppressWarnings("unchecked") public Iterator<Tuple2<KeyedWindow, SparkElement>> call(SparkElement se) throws Exception { final Iterable<Window> windows = windowing.assignWindowsToElement(new SparkElement( se.getWindow(), se.getTimestamp(), left ? Either.left(se.getElement()) : Either.right(se.getElement()))); return Iterators.transform(windows.iterator(), w -> new Tuple2<>( new KeyedWindow<>(w, se.getTimestamp(), keyExtractor.apply(se.getElement())), new SparkElement(w, se.getTimestamp(), se.getElement()))); }
@Override @SuppressWarnings("unchecked") public Iterator<Tuple2<KeyedWindow, Object>> call(SparkElement wel) throws Exception { Iterable<Window> windows = windowing.assignWindowsToElement(wel); return Iterators.transform(windows.iterator(), w -> new Tuple2<>( new KeyedWindow<>(w, wel.getTimestamp(), keyExtractor.apply(wel.getElement())), valueExtractor.apply(wel.getElement()))); } }
@Override public Iterator<SparkElement<WID, OUT>> call(SparkElement<WID, IN> elem) { final WID window = elem.getWindow(); final long timestamp = getTimestamp(elem); FunctionCollectorMem<OUT> collector = getContext(); // setup user collector collector.clear(); collector.setWindow(window); functor.apply(elem.getElement(), collector); // wrap output in WindowedElement return Iterators.transform(collector.getOutputIterator(), e -> new SparkElement<>(window, timestamp, e)); }
left.mapToPair( se -> { final Object key = operator.getLeftKeyExtractor().apply(se.getElement()); return new Tuple2<>( new BatchJoinKey<>(key, BatchJoinKey.Side.LEFT), right.mapToPair( se -> { final Object key = operator.getRightKeyExtractor().apply(se.getElement()); return new Tuple2<>( new BatchJoinKey<>(key, BatchJoinKey.Side.RIGHT), final long maxTimestamp = Math.max( first == null ? 0L : first.getTimestamp(), second == null ? 0L : second.getTimestamp()); collector.clear(); collector.setWindow(GlobalWindowing.Window.get()); .getJoiner() .apply( first == null ? null : first.getElement(), second == null ? null : second.getElement(), collector); return Iterators.transform( collector.getOutputIterator(), e -> new SparkElement<>( GlobalWindowing.Window.get(), maxTimestamp, Pair.of(t._1, e)));
@Override protected long getTimestamp(SparkElement<WID, IN> elem) { return evtTimeFn.extractTimestamp(elem.getElement()); } }
new SparkElement(kw.window(), kw.timestamp(), Pair.of(kw.key(), value)));
protected long getTimestamp(SparkElement<WID, IN> elem) { return elem.getTimestamp(); }
@Override public Iterator<Tuple2<KeyedWindow<W, KEY>, VALUE>> call(SparkElement<?, IN> wel) { final Iterable<W> windows = windowing.assignWindowsToElement(wel); return Iterators.transform( windows.iterator(), wid -> { final long stamp = Objects.requireNonNull(wid).maxTimestamp() - 1; return new Tuple2<>( new KeyedWindow<>(wid, stamp, keyExtractor.apply(wel.getElement())), valueExtractor.apply(wel.getElement())); }); } }
@Override public JavaRDD<?> translate(FlowUnfolder.InputOperator operator, SparkExecutorContext context) { // get original datasource from operator DataSource<?> ds = operator.output().getSource(); try { final long desiredSplitSize = context.getSettings() .getLong(DESIRED_SPLIT_SIZE, DEFAULT_DESIRED_SPLIT_SIZE); final Configuration conf = DataSourceInputFormat.configure( new Configuration(), ds, desiredSplitSize); @SuppressWarnings("unchecked") JavaPairRDD<Object, Object> pairs = context .getExecutionEnvironment() .newAPIHadoopRDD(conf, DataSourceInputFormat.class, Object.class, Object.class) .setName(operator.getName() + "::read"); // map values to WindowedElement return pairs .values() .map(v -> new SparkElement<>(GlobalWindowing.Window.get(), 0L, v)) .setName(operator.getName() + "::windowed-element"); } catch (IOException e) { throw new RuntimeException(e); } } }
final Window window = first == null ? second.getWindow() : first.getWindow(); final long maxTimestamp = Math.max( first == null ? window.maxTimestamp() - 1 : first.getTimestamp(), second == null ? window.maxTimestamp() - 1 : second.getTimestamp()); collector.clear(); collector.setWindow(window); .getJoiner() .apply( first == null ? null : first.getElement(), second == null ? null : second.getElement(), collector); return Iterators.transform( collector.getOutputIterator(), e -> new SparkElement<>(window, maxTimestamp, Pair.of(t._1.key(), e))); }, new LazyAccumulatorProvider(
new Tuple2<>(NullWritable.get(), el.getElement()));
@SuppressWarnings("unchecked") final OUT el = (OUT) t._2(); return new SparkElement<>(kw.window(), kw.timestamp(), Pair.of(kw.key(), el)); }) .setName(operator.getName() + "::wrap-in-spark-element"); final KeyedWindow<W, KEY> kw = t._1(); final OUT el = t._2(); return new SparkElement<>(kw.window(), kw.timestamp(), Pair.of(kw.key(), el)); }) .setName(operator.getName() + "::wrap-in-spark-element");