JavaPairDStream<Integer, Long> responseCodeCountDStream = accessLogsDStream.transformToPair( new Function<JavaRDD<ApacheAccessLog>, JavaPairRDD<Integer, Long>>() { public JavaPairRDD<Integer, Long> call(JavaRDD<ApacheAccessLog> rdd) { JavaPairDStream<String, Long> ipRawDStream = accessLogsDStream.transformToPair( new Function<JavaRDD<ApacheAccessLog>, JavaPairRDD<String, Long>>(){ public JavaPairRDD<String, Long> call(JavaRDD<ApacheAccessLog> rdd) { JavaPairDStream<String, Long> endpointCountsDStream = accessLogsDStream.transformToPair( new Function<JavaRDD<ApacheAccessLog>, JavaPairRDD<String, Long>>() { public JavaPairRDD<String, Long> call(JavaRDD<ApacheAccessLog> rdd) {
JavaPairDStream<Integer, Long> responseCodeCountTransform = accessLogsDStream.transformToPair( new Function<JavaRDD<ApacheAccessLog>, JavaPairRDD<Integer, Long>>() { public JavaPairRDD<Integer, Long> call(JavaRDD<ApacheAccessLog> logs) {
@Override protected SparkPairCollection<Object, Object> addJoinKey(StageSpec stageSpec, String inputStageName, SparkCollection<Object> inputCollection, StageStatisticsCollector collector) throws Exception { DynamicDriverContext dynamicDriverContext = new DynamicDriverContext(stageSpec, sec, collector); JavaDStream<Object> dStream = inputCollection.getUnderlying(); JavaPairDStream<Object, Object> result = dStream.transformToPair(new DynamicJoinOn<>(dynamicDriverContext, inputStageName)); return new PairDStreamCollection<>(sec, result); }
@Override public SparkCollection<RecordInfo<Object>> aggregate(StageSpec stageSpec, @Nullable Integer partitions, StageStatisticsCollector collector) { DynamicDriverContext dynamicDriverContext = new DynamicDriverContext(stageSpec, sec, collector); JavaPairDStream<Object, T> keyedCollection = stream.transformToPair(new DynamicAggregatorGroupBy<Object, T>(dynamicDriverContext)); JavaPairDStream<Object, Iterable<T>> groupedCollection = partitions == null ? keyedCollection.groupByKey() : keyedCollection.groupByKey(partitions); return wrap(groupedCollection.transform(new DynamicAggregatorAggregate<Object, T, Object>(dynamicDriverContext))); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
.transformToPair( (rdd, time) -> rdd.mapPartitions(