/** * Creates a JSON representation of the given dataflow plan. * * @param plan The dataflow plan. * @return The dataflow plan (prior to optimization) as a JSON string. */ public static String getPlanAsJSON(Plan plan) { List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan); return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks); } }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned * and all channels have a shipping strategy assigned. * * For more details on the optimization phase, see the comments for * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}. * * @param program The program to be translated. * @return The optimized plan. * * @throws CompilerException * Thrown, if the plan is invalid or the optimizer encountered an inconsistent * situation during the compilation process. */ public OptimizedPlan compile(Plan program) throws CompilerException { final OptimizerPostPass postPasser = getPostPassFromPlan(program); return compile(program, postPasser); }
public OptimizedPlan compileWithStats(Plan p) { return this.withStatsCompiler.compile(p); }
Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration); FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);
@Before public void setup() { Configuration flinkConf = new Configuration(); this.dataStats = new DataStatistics(); this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator(), flinkConf); this.withStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM); this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator(), flinkConf); this.noStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM); }
public OptimizedPlan compileNoStats(Plan p) { return this.noStatsCompiler.compile(p); }
public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices, ActorSystemLoader actorSystemLoader) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
/** * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned * and all channels have a shipping strategy assigned. * * For more details on the optimization phase, see the comments for * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}. * * @param program The program to be translated. * @return The optimized plan. * * @throws CompilerException * Thrown, if the plan is invalid or the optimizer encountered an inconsistent * situation during the compilation process. */ public OptimizedPlan compile(Plan program) throws CompilerException { final OptimizerPostPass postPasser = getPostPassFromPlan(program); return compile(program, postPasser); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan plan = createProgramPlan(jobName); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down */ public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = new LazyActorSystemLoader( highAvailabilityServices, Time.milliseconds(lookupTimeout.toMillis()), flinkConfig, log); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
/** * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned * and all channels have a shipping strategy assigned. * * For more details on the optimization phase, see the comments for * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}. * * @param program The program to be translated. * @return The optimized plan. * * @throws CompilerException * Thrown, if the plan is invalid or the optimizer encountered an inconsistent * situation during the compilation process. */ public OptimizedPlan compile(Plan program) throws CompilerException { final OptimizerPostPass postPasser = getPostPassFromPlan(program); return compile(program, postPasser); }
@Override public JobExecutionResult execute(String jobName) throws Exception { this.plan = createProgramPlan(jobName); this.previewPlan = Optimizer.createPreOptimizedPlan(plan); // do not go on with anything now! throw new OptimizerPlanEnvironment.ProgramAbortException(); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.configuration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan(null, false); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }
@Test public void testGetExecutionPlan() { try { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); InetAddress mockAddress = InetAddress.getLocalHost(); InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345); Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName()); config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }