How to use org.apache.flink.optimizer

  /**
   * Creates a JSON representation of the given dataflow plan.
   *
   * @param plan The dataflow plan.
   * @return The dataflow plan (prior to optimization) as a JSON string.
   */
  public static String getPlanAsJSON(Plan plan) {
    List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
    return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks);
  }
}

/**
 * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned
 * and all channels have a shipping strategy assigned.
 *
 * For more details on the optimization phase, see the comments for
 * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}.
 * 
 * @param program The program to be translated.
 * @return The optimized plan.
 *
 * @throws CompilerException
 *         Thrown, if the plan is invalid or the optimizer encountered an inconsistent
 *         situation during the compilation process.
 */
public OptimizedPlan compile(Plan program) throws CompilerException {
  final OptimizerPostPass postPasser = getPostPassFromPlan(program);
  return compile(program, postPasser);
}

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
  if (node instanceof SinkJoiner) {
    return new SinkJoinerPlanNode((SinkJoiner) node, in1, in2);
  } else {
    throw new CompilerException();
  }
}

  private JobGraph getJobGraph(final Plan plan) {
    final Optimizer pc = new Optimizer(new DataStatistics(), getConfiguration());
    final OptimizedPlan op = pc.compile(plan);
    final JobGraphGenerator jgg = new JobGraphGenerator();
    return jgg.compileJobGraph(op);
  }
}

@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
  Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
  OptimizedPlan optPlan = opt.compile(plan);
  return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan);
}

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
  final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
  Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
  pc.setDefaultParallelism(parallelism);
  OptimizedPlan op = pc.compile(plan);
  return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

private OptimizedPlan compileProgram(String jobName) {
  Plan p = createProgramPlan(jobName);
  Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
  return pc.compile(p);
}

@Before
public void setup() {
  Configuration flinkConf = new Configuration();
  this.dataStats = new DataStatistics();
  this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator(), flinkConf);
  this.withStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM);
  this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator(), flinkConf);
  this.noStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM);
}

public ClusterClient(
    Configuration flinkConfig,
    HighAvailabilityServices highAvailabilityServices,
    boolean sharedHaServices,
    ActorSystemLoader actorSystemLoader) {
  this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
  this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);
  this.timeout = AkkaUtils.getClientTimeout(flinkConfig);
  this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig);
  this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader);
  this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
  this.sharedHaServices = sharedHaServices;
}

public OptimizedPlan compileWithStats(Plan p) {
  return this.withStatsCompiler.compile(p);
}

  @Override
  public void postVisit(OptimizerNode visitable) {
    visitable.identifyDynamicPath(this.costWeight);

    // check that there is no nested iteration on the dynamic path
    if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
      throw new CompilerException("Nested iterations are currently not supported.");
    }
  }
}

private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) {
  if (pattern == DistributionPattern.ALL_TO_ALL) {
    return numSenders;
  } else if (pattern == DistributionPattern.POINTWISE) {
    if (numSenders != numReceivers) {
      if (numReceivers == 1) {
        return numSenders;
      }
      else if (numSenders == 1) {
        return 1;
      }
      else {
        throw new CompilerException("Error: A changing parallelism is currently " +
            "not supported between tasks within an iteration.");
      }
    } else {
      return 1;
    }
  } else {
    throw new CompilerException("Unknown distribution pattern for channels: " + pattern);
  }
}

public void setSourceStatistics(GenericDataSourceBase<?, ?> source, FileBaseStatistics stats) {
  final String key = CACHE_KEY + this.statCounter++;
  this.dataStats.cacheBaseStatistics(stats, key);
  source.setStatisticsKey(key);
}

  /**
   * Helpers to generate the JobGraph.
   */
  private static JobGraph getJobGraph(Plan plan) {
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    JobGraphGenerator jgg = new JobGraphGenerator();
    OptimizedPlan op = pc.compile(plan);
    return jgg.compileJobGraph(op);
  }
}

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
  final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
  Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
  pc.setDefaultParallelism(parallelism);
  OptimizedPlan op = pc.compile(plan);
  return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

public OptimizedPlan compileNoStats(Plan p) {
  return this.noStatsCompiler.compile(p);
}

public static boolean[] getDirections(Ordering o, int numFields) {
  final boolean[] dirs = o.getFieldSortDirections();
  if (dirs.length == numFields) {
    return dirs;
  } else if (dirs.length > numFields) {
    final boolean[] subSet = new boolean[numFields];
    System.arraycopy(dirs, 0, subSet, 0, numFields);
    return subSet;
  } else {
    throw new CompilerException();
  }
}

@Override
public String getExecutionPlan() throws Exception {
  Plan plan = createProgramPlan(null, false);
  this.optimizerPlan = compiler.compile(plan);
  // do not go on with anything now!
  throw new ProgramAbortException();
}

/**
 * Creates a new node for the bulk iteration.
 * 
 * @param iteration The bulk iteration the node represents.
 */
public BulkIterationNode(BulkIterationBase<?> iteration) {
  super(iteration);
  
  if (iteration.getMaximumNumberOfIterations() <= 0) {
    throw new CompilerException("BulkIteration must have a maximum number of iterations specified.");
  }
  
  int numIters = iteration.getMaximumNumberOfIterations();
  
  this.costWeight = (numIters > 0 && numIters < OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) ?
    numIters : OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; 
}

@Override
public JobExecutionResult execute(String jobName) throws Exception {
  Plan plan = createProgramPlan(jobName);
  this.optimizerPlan = compiler.compile(plan);
  // do not go on with anything now!
  throw new ProgramAbortException();
}