public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException { Logger log = LoggerFactory.getLogger(ClusterClient.class); if (parallelism > 0 && p.getDefaultParallelism() <= 0) { log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism); p.setDefaultParallelism(parallelism); } log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism()); return compiler.compile(p); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
final int defaultParallelism = program.getDefaultParallelism() > 0 ? program.getDefaultParallelism() : this.defaultParallelism;
if (plan.getDefaultParallelism() <= 0) { plan.setDefaultParallelism(defaultParallelism);
@Test public void testCorrectTranslation() { final String jobName = "Test JobName"; final int numIterations = 13; final int defaultParallelism = 133; final int iterationParallelism = 77; ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // ------------ construct the test program ------------------ { env.setParallelism(defaultParallelism); @SuppressWarnings("unchecked") DataSet<Tuple3<Double, Long, String>> initialDataSet = env.fromElements(new Tuple3<>(3.44, 5L, "abc")); IterativeDataSet<Tuple3<Double, Long, String>> bulkIteration = initialDataSet.iterate(numIterations); bulkIteration.setParallelism(iterationParallelism); // test that multiple iteration consumers are supported DataSet<Tuple3<Double, Long, String>> identity = bulkIteration .map(new IdentityMapper<Tuple3<Double, Long, String>>()); DataSet<Tuple3<Double, Long, String>> result = bulkIteration.closeWith(identity); result.output(new DiscardingOutputFormat<Tuple3<Double, Long, String>>()); result.writeAsText("/dev/null"); } Plan p = env.createProgramPlan(jobName); // ------------- validate the plan ---------------- BulkIterationBase<?> iteration = (BulkIterationBase<?>) p.getDataSinks().iterator().next().getInput(); assertEquals(jobName, p.getJobName()); assertEquals(defaultParallelism, p.getDefaultParallelism()); assertEquals(iterationParallelism, iteration.getParallelism()); }
assertEquals(defaultParallelism, p.getDefaultParallelism());
public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException { Logger log = LoggerFactory.getLogger(ClusterClient.class); if (parallelism > 0 && p.getDefaultParallelism() <= 0) { log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism); p.setDefaultParallelism(parallelism); } log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism()); return compiler.compile(p); }
public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException { Logger log = LoggerFactory.getLogger(ClusterClient.class); if (parallelism > 0 && p.getDefaultParallelism() <= 0) { log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism); p.setDefaultParallelism(parallelism); } log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism()); return compiler.compile(p); }
public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException { Logger log = LoggerFactory.getLogger(ClusterClient.class); if (parallelism > 0 && p.getDefaultParallelism() <= 0) { log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism); p.setDefaultParallelism(parallelism); } log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism()); return compiler.compile(p); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.configuration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.configuration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
if (plan.getDefaultParallelism() <= 0) { plan.setDefaultParallelism(defaultParallelism);
if (plan.getDefaultParallelism() <= 0) { plan.setDefaultParallelism(defaultParallelism);