@Test public void testSortPartitionParallelismChange() throws Exception { /* * Test sort partition with parallelism change */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .sortPartition(1, Order.DESCENDING).setParallelism(3) // change parallelism .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionByKeyField() throws Exception { /* * Test sort partition on key field */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input .sortPartition(1, Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Test public void testSortPartitionByFieldExpression() throws Exception { /* * Test sort partition on field expression */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper()).setParallelism(4) // parallelize input .sortPartition("f1", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionByTwoFieldExpressions() throws Exception { /* * Test sort partition on two field expressions */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(2); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple5<Integer, Long, Integer, String, Long>>()).setParallelism(2) // parallelize input .sortPartition("f4", Order.ASCENDING) .sortPartition("f2", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple5Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionByNestedFieldExpression() throws Exception { /* * Test sort partition on nested field expressions */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple2<Tuple2<Integer, Integer>, String>>()).setParallelism(3) // parallelize input .sortPartition("f0.f1", Order.ASCENDING) .sortPartition("f1", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new NestedTupleChecker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionPojoByNestedFieldExpression() throws Exception { /* * Test sort partition on field expression */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataSet<POJO> ds = CollectionDataSets.getMixedPojoDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<POJO>()).setParallelism(1) // parallelize input .sortPartition("nestedTupleWithCustom.f1.myString", Order.ASCENDING) .sortPartition("number", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new PojoChecker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionByTwoKeyFields() throws Exception { /* * Test sort partition on two key fields */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(2); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple5<Integer, Long, Integer, String, Long>>()).setParallelism(2) // parallelize input .sortPartition(4, Order.ASCENDING) .sortPartition(2, Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple5Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionWithKeySelector1() throws Exception { /* * Test sort partition on an extracted key */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input .sortPartition(new KeySelector<Tuple3<Integer, Long, String>, Long>() { @Override public Long getKey(Tuple3<Integer, Long, String> value) throws Exception { return value.f1; } }, Order.ASCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3AscendingChecker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
@Test public void testSortPartitionWithKeySelector2() throws Exception { /* * Test sort partition on an extracted key */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input .sortPartition(new KeySelector<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>>() { @Override public Tuple2<Integer, Long> getKey(Tuple3<Integer, Long, String> value) throws Exception { return new Tuple2<>(value.f0, value.f1); } }, Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }