@SuppressWarnings("unchecked") @Test public void testMap() { List<List<String>> inputData = Arrays.asList( Arrays.asList("hello", "world"), Arrays.asList("goodnight", "moon")); List<List<Integer>> expected = Arrays.asList( Arrays.asList(5,5), Arrays.asList(9,4)); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> letterCount = stream.map(String::length); JavaTestUtils.attachTestOutputStream(letterCount); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testRepartitionFewerPartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 4); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(2); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(2, rdd.size()); Assert.assertEquals(10, rdd.get(0).size() + rdd.get(1).size()); } }
@SuppressWarnings("unchecked") @Test public void testTextFileStream() throws IOException { File testDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); List<List<String>> expected = fileTestPrepare(testDir); JavaDStream<String> input = ssc.textFileStream(testDir.toString()); JavaTestUtils.attachTestOutputStream(input); List<List<String>> result = JavaTestUtils.runStreams(ssc, 1, 1); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testContextState() { List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1, 2, 3, 4)); Assert.assertEquals(StreamingContextState.INITIALIZED, ssc.getState()); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaTestUtils.attachTestOutputStream(stream); Assert.assertEquals(StreamingContextState.INITIALIZED, ssc.getState()); ssc.start(); Assert.assertEquals(StreamingContextState.ACTIVE, ssc.getState()); ssc.stop(); Assert.assertEquals(StreamingContextState.STOPPED, ssc.getState()); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testTextFileStream() throws IOException { File testDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); List<List<String>> expected = fileTestPrepare(testDir); JavaDStream<String> input = ssc.textFileStream(testDir.toString()); JavaTestUtils.attachTestOutputStream(input); List<List<String>> result = JavaTestUtils.runStreams(ssc, 1, 1); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testContextState() { List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1, 2, 3, 4)); Assert.assertEquals(StreamingContextState.INITIALIZED, ssc.getState()); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaTestUtils.attachTestOutputStream(stream); Assert.assertEquals(StreamingContextState.INITIALIZED, ssc.getState()); ssc.start(); Assert.assertEquals(StreamingContextState.ACTIVE, ssc.getState()); ssc.stop(); Assert.assertEquals(StreamingContextState.STOPPED, ssc.getState()); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testFilter() { List<List<String>> inputData = Arrays.asList( Arrays.asList("giants", "dodgers"), Arrays.asList("yankees", "red sox")); List<List<String>> expected = Arrays.asList( Arrays.asList("giants"), Arrays.asList("yankees")); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<String> filtered = stream.filter(s -> s.contains("a")); JavaTestUtils.attachTestOutputStream(filtered); List<List<String>> result = JavaTestUtils.runStreams(ssc, 2, 2); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testRepartitionMorePartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 2); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(4); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(4, rdd.size()); Assert.assertEquals( 10, rdd.get(0).size() + rdd.get(1).size() + rdd.get(2).size() + rdd.get(3).size()); } }
@SuppressWarnings("unchecked") @Test public void testFileStream() throws IOException { File testDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); List<List<String>> expected = fileTestPrepare(testDir); JavaPairInputDStream<LongWritable, Text> inputStream = ssc.fileStream( testDir.toString(), LongWritable.class, Text.class, TextInputFormat.class, v1 -> Boolean.TRUE, true); JavaDStream<String> test = inputStream.map(v1 -> v1._2().toString()); JavaTestUtils.attachTestOutputStream(test); List<List<String>> result = JavaTestUtils.runStreams(ssc, 1, 1); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@Test public void testFilter() { List<List<String>> inputData = Arrays.asList( Arrays.asList("giants", "dodgers"), Arrays.asList("yankees", "red sox")); List<List<String>> expected = Arrays.asList( Arrays.asList("giants"), Arrays.asList("yankees")); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<String> filtered = stream.filter(s -> s.contains("a")); JavaTestUtils.attachTestOutputStream(filtered); List<List<String>> result = JavaTestUtils.runStreams(ssc, 2, 2); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testRepartitionMorePartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 2); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(4); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(4, rdd.size()); Assert.assertEquals( 10, rdd.get(0).size() + rdd.get(1).size() + rdd.get(2).size() + rdd.get(3).size()); } }
@SuppressWarnings("unchecked") @Test public void testFileStream() throws IOException { File testDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); List<List<String>> expected = fileTestPrepare(testDir); JavaPairInputDStream<LongWritable, Text> inputStream = ssc.fileStream( testDir.toString(), LongWritable.class, Text.class, TextInputFormat.class, v1 -> Boolean.TRUE, true); JavaDStream<String> test = inputStream.map(v1 -> v1._2().toString()); JavaTestUtils.attachTestOutputStream(test); List<List<String>> result = JavaTestUtils.runStreams(ssc, 1, 1); assertOrderInvariantEquals(expected, result); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Test public void testMap() { List<List<String>> inputData = Arrays.asList( Arrays.asList("hello", "world"), Arrays.asList("goodnight", "moon")); List<List<Integer>> expected = Arrays.asList( Arrays.asList(5, 5), Arrays.asList(9, 4)); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> letterCount = stream.map(String::length); JavaTestUtils.attachTestOutputStream(letterCount); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testRepartitionFewerPartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 4); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(2); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(2, rdd.size()); Assert.assertEquals(10, rdd.get(0).size() + rdd.get(1).size()); } }
@Test public void testMap() { List<List<String>> inputData = Arrays.asList( Arrays.asList("hello", "world"), Arrays.asList("goodnight", "moon")); List<List<Integer>> expected = Arrays.asList( Arrays.asList(5, 5), Arrays.asList(9, 4)); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> letterCount = stream.map(String::length); JavaTestUtils.attachTestOutputStream(letterCount); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); assertOrderInvariantEquals(expected, result); }
@Test public void testFilter() { List<List<String>> inputData = Arrays.asList( Arrays.asList("giants", "dodgers"), Arrays.asList("yankees", "red sox")); List<List<String>> expected = Arrays.asList( Arrays.asList("giants"), Arrays.asList("yankees")); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<String> filtered = stream.filter(s -> s.contains("a")); JavaTestUtils.attachTestOutputStream(filtered); List<List<String>> result = JavaTestUtils.runStreams(ssc, 2, 2); assertOrderInvariantEquals(expected, result); }