@Test public void testBulkIteration() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(10); DataSet<Integer> result = iteration.closeWith(iteration.map(new AddSuperstepNumberMapper())); List<Integer> collected = new ArrayList<Integer>(); result.output(new LocalCollectionOutputFormat<Integer>(collected)); env.execute(); assertEquals(1, collected.size()); assertEquals(56, collected.get(0).intValue()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
env.registerCachedFile(resultPath, testName); IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2); solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() { @Override
@Test public void testBulkIterationWithTerminationCriterion() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(100); DataSet<Integer> iterationResult = iteration.map(new AddSuperstepNumberMapper()); DataSet<Integer> terminationCriterion = iterationResult.filter(new FilterFunction<Integer>() { public boolean filter(Integer value) { return value < 50; } }); List<Integer> collected = new ArrayList<Integer>(); iteration.closeWith(iterationResult, terminationCriterion) .output(new LocalCollectionOutputFormat<Integer>(collected)); env.execute(); assertEquals(1, collected.size()); assertEquals(56, collected.get(0).intValue()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); IterativeDataSet<Long> iteration = env.generateSequence(1, 10).iterate(100); iteration.closeWith(iteration) .output(new LocalCollectionOutputFormat<Long>(result)); env.execute(); }
@Test public void testIdentityIteration() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(43); IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100); iteration.closeWith(iteration).output(new DiscardingOutputFormat<Long>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); new JobGraphGenerator().compileJobGraph(op); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(NUM_SUBTASKS); IterativeDataSet<Integer> iteration = env.fromElements(1, 2, 3).iterate(NUM_ITERATIONS); iteration.closeWith(iteration.reduceGroup(new SumReducer())).output(new DiscardingOutputFormat<Integer>()); Assert.assertEquals(NUM_ITERATIONS * 6, (int) env.execute().getAccumulatorResult(ACC_NAME)); }
env.setParallelism(43); IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);