public JavaPairDStream<Long, Integer> readStats(JavaStreamingContext jssc, String inputDirectory) { // Note: This example doesn't work until Spark 1.2 JavaPairDStream<LongWritable, Text> input = jssc.fileStream(inputDirectory, LongWritable.class, Text.class, TextInputFormat.class); // convert the input from Writables to native types JavaPairDStream<Long, Integer> usefulInput = input.mapToPair( new PairFunction<Tuple2<LongWritable, Text>, Long, Integer>() { public Tuple2<Long, Integer> call(Tuple2<LongWritable, Text> input) { return new Tuple2(input._1().get(), Integer.parseInt(input._2().toString())); } }); return usefulInput; }
JavaPairDStream<Text, LongWritable> writableDStream = ipDStream.mapToPair( new PairFunction<Tuple2<String, Long>, Text, LongWritable>() { public Tuple2<Text, LongWritable> call(Tuple2<String, Long> e) {
@Test public void testPairMap() { // Maps pair -> pair of different type List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<Integer, String>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>(1, "california"), new Tuple2<>(3, "california"), new Tuple2<>(4, "new york"), new Tuple2<>(1, "new york")), Arrays.asList( new Tuple2<>(5, "california"), new Tuple2<>(5, "california"), new Tuple2<>(3, "new york"), new Tuple2<>(1, "new york"))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<Integer, String> reversed = pairStream.mapToPair(Tuple2::swap); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testPairMap() { // Maps pair -> pair of different type List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<Integer, String>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>(1, "california"), new Tuple2<>(3, "california"), new Tuple2<>(4, "new york"), new Tuple2<>(1, "new york")), Arrays.asList( new Tuple2<>(5, "california"), new Tuple2<>(5, "california"), new Tuple2<>(3, "new york"), new Tuple2<>(1, "new york"))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<Integer, String> reversed = pairStream.mapToPair(Tuple2::swap); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testPairMap() { // Maps pair -> pair of different type List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<Integer, String>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>(1, "california"), new Tuple2<>(3, "california"), new Tuple2<>(4, "new york"), new Tuple2<>(1, "new york")), Arrays.asList( new Tuple2<>(5, "california"), new Tuple2<>(5, "california"), new Tuple2<>(3, "new york"), new Tuple2<>(1, "new york"))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<Integer, String> reversed = pairStream.mapToPair(Tuple2::swap); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testPairMap() { // Maps pair -> pair of different type List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<Integer, String>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>(1, "california"), new Tuple2<>(3, "california"), new Tuple2<>(4, "new york"), new Tuple2<>(1, "new york")), Arrays.asList( new Tuple2<>(5, "california"), new Tuple2<>(5, "california"), new Tuple2<>(3, "new york"), new Tuple2<>(1, "new york"))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<Integer, String> reversed = pairStream.mapToPair(Tuple2::swap); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
JavaPairDStream<Integer, String> swappedCounts = counts.mapToPair( (PairFunction<Tuple2<String, Integer>, Integer, String>) in -> in.swap() );