How to use

ReduceNode

in

org.apache.flink.optimizer.dag

  public ReduceNode getCombinerUtilityNode() {
    if (this.preReduceUtilityNode == null) {
      this.preReduceUtilityNode = new ReduceNode(this);
      
      // we conservatively assume the combiner returns the same data size as it consumes 
      this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
      this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
    }
    return this.preReduceUtilityNode;
  }
}

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
  Channel toReducer = in;
  if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
      (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) {
    if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
      LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
    }
  } else if (combinerStrategy != DriverStrategy.NONE) {
    // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
    Channel toCombiner = new Channel(in.getSource());
    toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
    // create an input node for combine with same parallelism as input node
    ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
    combinerNode.setParallelism(in.getSource().getParallelism());
    SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
              "Combine ("+node.getOperator().getName()+")", toCombiner,
              this.combinerStrategy, this.keyList);
    combiner.setCosts(new Costs(0, 0));
    combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
    toReducer = new Channel(combiner);
    toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
                  in.getShipStrategySortOrder(), in.getDataExchangeMode());
    toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
  }
  return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer,
    DriverStrategy.SORTED_REDUCE, this.keyList);
}

setParallelism(1);

n = new ReduceNode((ReduceOperatorBase<?, ?>) c);

setParallelism(1);

n = new ReduceNode((ReduceOperatorBase<?, ?>) c);

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
  if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
    // locally connected, directly instantiate
    return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
                    in, DriverStrategy.ALL_REDUCE);
  } else {
    // non forward case.plug in a combiner
    Channel toCombiner = new Channel(in.getSource());
    toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
    
    // create an input node for combine with same parallelism as input node
    ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
    combinerNode.setParallelism(in.getSource().getParallelism());
    SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
        "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
    combiner.setCosts(new Costs(0, 0));
    combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
    
    Channel toReducer = new Channel(combiner);
    toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
                  in.getShipStrategySortOrder(), in.getDataExchangeMode());
    toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
                  in.getLocalStrategySortOrder());
    return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
                    toReducer, DriverStrategy.ALL_REDUCE);
  }
}

  public ReduceNode getCombinerUtilityNode() {
    if (this.preReduceUtilityNode == null) {
      this.preReduceUtilityNode = new ReduceNode(this);
      
      // we conservatively assume the combiner returns the same data size as it consumes 
      this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
      this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
    }
    return this.preReduceUtilityNode;
  }
}

setParallelism(1);

n = new ReduceNode((ReduceOperatorBase<?, ?>) c);

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
  Channel toReducer = in;
  if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
      (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) {
    if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
      LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
    }
  } else if (combinerStrategy != DriverStrategy.NONE) {
    // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
    Channel toCombiner = new Channel(in.getSource());
    toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
    // create an input node for combine with same parallelism as input node
    ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
    combinerNode.setParallelism(in.getSource().getParallelism());
    SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
              "Combine ("+node.getOperator().getName()+")", toCombiner,
              this.combinerStrategy, this.keyList);
    combiner.setCosts(new Costs(0, 0));
    combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
    toReducer = new Channel(combiner);
    toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
                  in.getShipStrategySortOrder(), in.getDataExchangeMode());
    toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
  }
  return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer,
    DriverStrategy.SORTED_REDUCE, this.keyList);
}

  public ReduceNode getCombinerUtilityNode() {
    if (this.preReduceUtilityNode == null) {
      this.preReduceUtilityNode = new ReduceNode(this);
      
      // we conservatively assume the combiner returns the same data size as it consumes 
      this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
      this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
    }
    return this.preReduceUtilityNode;
  }
}

setParallelism(1);

n = new ReduceNode((ReduceOperatorBase<?, ?>) c);

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
  Channel toReducer = in;
  if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
      (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) {
    if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
      LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
    }
  } else if (combinerStrategy != DriverStrategy.NONE) {
    // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
    Channel toCombiner = new Channel(in.getSource());
    toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
    // create an input node for combine with same parallelism as input node
    ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
    combinerNode.setParallelism(in.getSource().getParallelism());
    SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
              "Combine ("+node.getOperator().getName()+")", toCombiner,
              this.combinerStrategy, this.keyList);
    combiner.setCosts(new Costs(0, 0));
    combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
    toReducer = new Channel(combiner);
    toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
                  in.getShipStrategySortOrder(), in.getDataExchangeMode());
    toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
  }
  return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer,
    DriverStrategy.SORTED_REDUCE, this.keyList);
}

  public ReduceNode getCombinerUtilityNode() {
    if (this.preReduceUtilityNode == null) {
      this.preReduceUtilityNode = new ReduceNode(this);
      
      // we conservatively assume the combiner returns the same data size as it consumes 
      this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
      this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
    }
    return this.preReduceUtilityNode;
  }
}

setParallelism(1);

n = new ReduceNode((ReduceOperatorBase<?, ?>) c);

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
  Channel toReducer = in;
  if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
      (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) {
    if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
      LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
    }
  } else if (combinerStrategy != DriverStrategy.NONE) {
    // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
    Channel toCombiner = new Channel(in.getSource());
    toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
    // create an input node for combine with same parallelism as input node
    ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
    combinerNode.setParallelism(in.getSource().getParallelism());
    SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
              "Combine ("+node.getOperator().getName()+")", toCombiner,
              this.combinerStrategy, this.keyList);
    combiner.setCosts(new Costs(0, 0));
    combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
    toReducer = new Channel(combiner);
    toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
                  in.getShipStrategySortOrder(), in.getDataExchangeMode());
    toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
  }
  return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer,
    DriverStrategy.SORTED_REDUCE, this.keyList);
}

  public ReduceNode getCombinerUtilityNode() {
    if (this.preReduceUtilityNode == null) {
      this.preReduceUtilityNode = new ReduceNode(this);
      
      // we conservatively assume the combiner returns the same data size as it consumes 
      this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
      this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
    }
    return this.preReduceUtilityNode;
  }
}