@Override @SuppressWarnings("unchecked") public Iterator<Tuple2<KeyedWindow, SparkElement>> call(SparkElement se) throws Exception { final Iterable<Window> windows = windowing.assignWindowsToElement(new SparkElement( se.getWindow(), se.getTimestamp(), left ? Either.left(se.getElement()) : Either.right(se.getElement()))); return Iterators.transform(windows.iterator(), w -> new Tuple2<>( new KeyedWindow<>(w, se.getTimestamp(), keyExtractor.apply(se.getElement())), new SparkElement(w, se.getTimestamp(), se.getElement()))); }
new SparkElement(kw.window(), kw.timestamp(), Pair.of(kw.key(), value)));
@Override public Iterator<SparkElement<WID, OUT>> call(SparkElement<WID, IN> elem) { final WID window = elem.getWindow(); final long timestamp = getTimestamp(elem); FunctionCollectorMem<OUT> collector = getContext(); // setup user collector collector.clear(); collector.setWindow(window); functor.apply(elem.getElement(), collector); // wrap output in WindowedElement return Iterators.transform(collector.getOutputIterator(), e -> new SparkElement<>(window, timestamp, e)); }
@Override public JavaRDD<?> translate(FlowUnfolder.InputOperator operator, SparkExecutorContext context) { // get original datasource from operator DataSource<?> ds = operator.output().getSource(); try { final long desiredSplitSize = context.getSettings() .getLong(DESIRED_SPLIT_SIZE, DEFAULT_DESIRED_SPLIT_SIZE); final Configuration conf = DataSourceInputFormat.configure( new Configuration(), ds, desiredSplitSize); @SuppressWarnings("unchecked") JavaPairRDD<Object, Object> pairs = context .getExecutionEnvironment() .newAPIHadoopRDD(conf, DataSourceInputFormat.class, Object.class, Object.class) .setName(operator.getName() + "::read"); // map values to WindowedElement return pairs .values() .map(v -> new SparkElement<>(GlobalWindowing.Window.get(), 0L, v)) .setName(operator.getName() + "::windowed-element"); } catch (IOException e) { throw new RuntimeException(e); } } }
return Iterators.transform( collector.getOutputIterator(), e -> new SparkElement<>(window, maxTimestamp, Pair.of(t._1.key(), e))); }, new LazyAccumulatorProvider(
collector.getOutputIterator(), e -> new SparkElement<>( GlobalWindowing.Window.get(), maxTimestamp, Pair.of(t._1, e))); },
@SuppressWarnings("unchecked") final OUT el = (OUT) t._2(); return new SparkElement<>(kw.window(), kw.timestamp(), Pair.of(kw.key(), el)); }) .setName(operator.getName() + "::wrap-in-spark-element"); final KeyedWindow<W, KEY> kw = t._1(); final OUT el = t._2(); return new SparkElement<>(kw.window(), kw.timestamp(), Pair.of(kw.key(), el)); }) .setName(operator.getName() + "::wrap-in-spark-element");