private <T, U> PairDStreamCollection<T, U> wrap(JavaPairDStream<T, U> pairStream) { return new PairDStreamCollection<>(sec, pairStream); } }
@Override public <T> SparkPairCollection<K, T> mapValues(Function<V, T> function) { return wrap(pairStream.mapValues(function)); }
@Override public <K, V> SparkPairCollection<K, V> flatMapToPair(PairFlatMapFunction<T, K, V> function) { return new PairDStreamCollection<>(sec, stream.flatMapToPair(function)); }
@SuppressWarnings("unchecked") @Override public <T> SparkPairCollection<K, Tuple2<Optional<V>, Optional<T>>> fullOuterJoin(SparkPairCollection<K, T> other) { return wrap(StreamingCompat.fullOuterJoin(pairStream, (JavaPairDStream<K, T>) other.getUnderlying())); }
@Override protected SparkPairCollection<Object, Object> addJoinKey(StageSpec stageSpec, String inputStageName, SparkCollection<Object> inputCollection, StageStatisticsCollector collector) throws Exception { DynamicDriverContext dynamicDriverContext = new DynamicDriverContext(stageSpec, sec, collector); JavaDStream<Object> dStream = inputCollection.getUnderlying(); JavaPairDStream<Object, Object> result = dStream.transformToPair(new DynamicJoinOn<>(dynamicDriverContext, inputStageName)); return new PairDStreamCollection<>(sec, result); }
@SuppressWarnings("unchecked") @Override public <T> SparkPairCollection<K, Tuple2<V, T>> join(SparkPairCollection<K, T> other, int numPartitions) { return wrap(pairStream.join((JavaPairDStream<K, T>) other.getUnderlying(), numPartitions)); }
@SuppressWarnings("unchecked") @Override public <T> SparkPairCollection<K, Tuple2<V, Optional<T>>> leftOuterJoin(SparkPairCollection<K, T> other) { return wrap(StreamingCompat.leftOuterJoin(pairStream, (JavaPairDStream<K, T>) other.getUnderlying())); }
@SuppressWarnings("unchecked") @Override public <T> SparkPairCollection<K, Tuple2<V, T>> join(SparkPairCollection<K, T> other) { return wrap(pairStream.join((JavaPairDStream<K, T>) other.getUnderlying())); }
@SuppressWarnings("unchecked") @Override public <T> SparkPairCollection<K, Tuple2<V, Optional<T>>> leftOuterJoin(SparkPairCollection<K, T> other, int numPartitions) { return wrap( StreamingCompat.leftOuterJoin(pairStream, (JavaPairDStream<K, T>) other.getUnderlying(), numPartitions)); }
@SuppressWarnings("unchecked") @Override public <T> SparkPairCollection<K, Tuple2<Optional<V>, Optional<T>>> fullOuterJoin(SparkPairCollection<K, T> other, int numPartitions) { return wrap( StreamingCompat.fullOuterJoin(pairStream, (JavaPairDStream<K, T>) other.getUnderlying(), numPartitions)); }