private JavaDStream<?> repartition(JavaDStream<?> stream) { int numPartitions = config.getInt(REPARTITION_NUM_PARTITIONS_PROPERTY); return stream.repartition(numPartitions); }
@SuppressWarnings("unchecked") @Test public void testRepartitionMorePartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 2); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(4); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(4, rdd.size()); Assert.assertEquals( 10, rdd.get(0).size() + rdd.get(1).size() + rdd.get(2).size() + rdd.get(3).size()); } }
@SuppressWarnings("unchecked") @Test public void testRepartitionMorePartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 2); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(4); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(4, rdd.size()); Assert.assertEquals( 10, rdd.get(0).size() + rdd.get(1).size() + rdd.get(2).size() + rdd.get(3).size()); } }
@SuppressWarnings("unchecked") @Test public void testRepartitionFewerPartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 4); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(2); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(2, rdd.size()); Assert.assertEquals(10, rdd.get(0).size() + rdd.get(1).size()); } }
@SuppressWarnings("unchecked") @Test public void testRepartitionFewerPartitions() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 4); JavaDStreamLike<Integer,JavaDStream<Integer>,JavaRDD<Integer>> repartitioned = stream.repartition(2); JavaTestUtils.attachTestOutputStream(repartitioned); List<List<List<Integer>>> result = JavaTestUtils.runStreamsWithPartitions(ssc, 2, 2); Assert.assertEquals(2, result.size()); for (List<List<Integer>> rdd : result) { Assert.assertEquals(2, rdd.size()); Assert.assertEquals(10, rdd.get(0).size() + rdd.get(1).size()); } }