org.apache.flink.api.common.operators.base.DeltaIterationBase java code examples

@SuppressWarnings("unchecked")
private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration) throws Exception {
  Operator<?> solutionInput = iteration.getInitialSolutionSet();
  Operator<?> worksetInput = iteration.getInitialWorkset();
  if (solutionInput == null) {
    throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial solution set.");
    throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial workset.");
  if (iteration.getSolutionSetDelta() == null) {
    throw new InvalidProgramException("The iteration " + iteration.getName() + " has no solution set delta defined (is not closed).");
  if (iteration.getNextWorkset() == null) {
    throw new InvalidProgramException("The iteration " + iteration.getName() + " has no workset defined (is not closed).");
  iteration.getSolutionSetDelta().accept(dynCollector);
  iteration.getNextWorkset().accept(dynCollector);
  BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo();
  TypeInformation<?> solutionType = operatorInfo.getFirstInputType();
  int[] keyColumns = iteration.getSolutionSetKeyFields();
  boolean[] inputOrderings = new boolean[keyColumns.length];
  TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0, executionConfig);
  for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
    aggregators.put(a.getName(), a.getAggregator());
  String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
  ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();

headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());

/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
  super(iteration);
  
  final int[] ssKeys = iteration.getSolutionSetKeyFields();
  if (ssKeys == null || ssKeys.length == 0) {
    throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
  }
  this.solutionSetKeyFields = new FieldList(ssKeys);
  this.partitionedProperties = new GlobalProperties();
  this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
  
  int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
    iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
    
  if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
    weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
  }
  this.costWeight = weight; 
  
  this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}

private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) {
  @SuppressWarnings("unchecked")
  DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd;
  DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead();
  String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName();
  DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()),
      iterationEnd.getKeyPositions(), name);
  iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations());
  if (iterationHead.getParallelism() > 0) {
    iterationOperator.setParallelism(iterationHead.getParallelism());
  }
  DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet();
  DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset();
  translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet());
  translated.put(worksetPlaceHolder, iterationOperator.getWorkset());
  Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet());
  Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset());
  iterationOperator.setNextWorkset(translatedWorkset);
  iterationOperator.setSolutionSetDelta(translatedSolutionSet);
  iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet()));
  iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));
  // register all aggregators
  iterationOperator.getAggregators().addAll(iterationHead.getAggregators());
  iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged());
  return iterationOperator;
}

  iter.getNextWorkset().accept(wsf);
  if (!wsf.hasFoundWorkset()) {
    throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
iter.getSolutionSetDelta().accept(recursiveCreator);
final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
iter.getNextWorkset().accept(recursiveCreator);
SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
  solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());

assertEquals(numIterations, iteration.getMaximumNumberOfIterations());
assertArrayEquals(iterationKeys, iteration.getSolutionSetKeyFields());
assertEquals(iterationParallelism, iteration.getParallelism());
assertEquals(iterationName, iteration.getName());
MapOperatorBase<?, ?, ?> nextWorksetMapper = (MapOperatorBase<?, ?, ?>) iteration.getNextWorkset();
InnerJoinOperatorBase<?, ?, ?, ?> solutionSetJoin = (InnerJoinOperatorBase<?, ?, ?, ?>) iteration.getSolutionSetDelta();
InnerJoinOperatorBase<?, ?, ?, ?> worksetSelfJoin = (InnerJoinOperatorBase<?, ?, ?, ?>) solutionSetJoin.getFirstInput();
MapOperatorBase<?, ?, ?> worksetMapper = (MapOperatorBase<?, ?, ?>) worksetSelfJoin.getFirstInput();
assertEquals(aggregatorName, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());

  final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
  if (maxNumIterations < 1) {
    throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();

@Override
public boolean preVisit(Operator<?> visitable) {
  if (this.seen.add(visitable)) {
    // add to  the map
    final String name = visitable.getName();
    List<Operator<?>> list = this.map.get(name);
    if (list == null) {
      list = new ArrayList<Operator<?>>(2);
      this.map.put(name, list);
    }
    list.add(visitable);
    
    // recurse into bulk iterations
    if (visitable instanceof BulkIterationBase) {
      ((BulkIterationBase) visitable).getNextPartialSolution().accept(this);
    } else if (visitable instanceof DeltaIterationBase) {
      ((DeltaIterationBase) visitable).getSolutionSetDelta().accept(this);
      ((DeltaIterationBase) visitable).getNextWorkset().accept(this);
    }
    
    return true;
  } else {
    return false;
  }
}

iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
    iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));

  iter.getNextWorkset().accept(wsf);
  if (!wsf.hasFoundWorkset()) {
    throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
iter.getSolutionSetDelta().accept(recursiveCreator);
final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
iter.getNextWorkset().accept(recursiveCreator);
SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
  solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());

private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) {
  @SuppressWarnings("unchecked")
  DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd;
  DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead();
  String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName();
  DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()),
      iterationEnd.getKeyPositions(), name);
  iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations());
  if (iterationHead.getParallelism() > 0) {
    iterationOperator.setParallelism(iterationHead.getParallelism());
  }
  DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet();
  DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset();
  translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet());
  translated.put(worksetPlaceHolder, iterationOperator.getWorkset());
  Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet());
  Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset());
  iterationOperator.setNextWorkset(translatedWorkset);
  iterationOperator.setSolutionSetDelta(translatedSolutionSet);
  iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet()));
  iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));
  // register all aggregators
  iterationOperator.getAggregators().addAll(iterationHead.getAggregators());
  iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged());
  return iterationOperator;
}

  final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
  if (maxNumIterations < 1) {
    throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();

/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
  super(iteration);
  
  final int[] ssKeys = iteration.getSolutionSetKeyFields();
  if (ssKeys == null || ssKeys.length == 0) {
    throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
  }
  this.solutionSetKeyFields = new FieldList(ssKeys);
  this.partitionedProperties = new GlobalProperties();
  this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
  
  int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
    iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
    
  if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
    weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
  }
  this.costWeight = weight; 
  
  this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}

iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
    iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));

@SuppressWarnings("unchecked")
private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration) throws Exception {
  Operator<?> solutionInput = iteration.getInitialSolutionSet();
  Operator<?> worksetInput = iteration.getInitialWorkset();
  if (solutionInput == null) {
    throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial solution set.");
    throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial workset.");
  if (iteration.getSolutionSetDelta() == null) {
    throw new InvalidProgramException("The iteration " + iteration.getName() + " has no solution set delta defined (is not closed).");
  if (iteration.getNextWorkset() == null) {
    throw new InvalidProgramException("The iteration " + iteration.getName() + " has no workset defined (is not closed).");
  iteration.getSolutionSetDelta().accept(dynCollector);
  iteration.getNextWorkset().accept(dynCollector);
  BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo();
  TypeInformation<?> solutionType = operatorInfo.getFirstInputType();
  int[] keyColumns = iteration.getSolutionSetKeyFields();
  boolean[] inputOrderings = new boolean[keyColumns.length];
  TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0, executionConfig);
  for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
    aggregators.put(a.getName(), a.getAggregator());
  String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
  ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();

  iter.getNextWorkset().accept(wsf);
  if (!wsf.hasFoundWorkset()) {
    throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
iter.getSolutionSetDelta().accept(recursiveCreator);
final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
iter.getNextWorkset().accept(recursiveCreator);
SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
  solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());

private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) {
  @SuppressWarnings("unchecked")
  DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd;
  DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead();
  String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName();
  DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()),
      iterationEnd.getKeyPositions(), name);
  iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations());
  if (iterationHead.getParallelism() > 0) {
    iterationOperator.setParallelism(iterationHead.getParallelism());
  }
  DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet();
  DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset();
  translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet());
  translated.put(worksetPlaceHolder, iterationOperator.getWorkset());
  Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet());
  Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset());
  iterationOperator.setNextWorkset(translatedWorkset);
  iterationOperator.setSolutionSetDelta(translatedSolutionSet);
  iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet()));
  iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));
  // register all aggregators
  iterationOperator.getAggregators().addAll(iterationHead.getAggregators());
  iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged());
  return iterationOperator;
}

  final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
  if (maxNumIterations < 1) {
    throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();

/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
  super(iteration);
  
  final int[] ssKeys = iteration.getSolutionSetKeyFields();
  if (ssKeys == null || ssKeys.length == 0) {
    throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
  }
  this.solutionSetKeyFields = new FieldList(ssKeys);
  this.partitionedProperties = new GlobalProperties();
  this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
  
  int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
    iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
    
  if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
    weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
  }
  this.costWeight = weight; 
  
  this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}

iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
    iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));

Javadoc

A DeltaIteration is similar to a BulkIterationBase, but maintains state across the individual iteration steps. The state is called the solution set, can be obtained via #getSolutionSet(), and be accessed by joining (or CoGrouping) with it. The solution set is updated by producing a delta for it, which is merged into the solution set at the end of each iteration step.

The delta iteration must be closed by setting a delta for the solution set ( #setSolutionSetDelta(org.apache.flink.api.common.operators.Operator)) and the new workset (the data set that will be fed back, #setNextWorkset(org.apache.flink.api.common.operators.Operator)). The DeltaIteration itself represents the result after the iteration has terminated. Delta iterations terminate when the feed back data set (the workset) is empty. In addition, a maximum number of steps is given as a fall back termination guard.

Elements in the solution set are uniquely identified by a key. When merging the solution set delta, contained elements with the same key are replaced.

This class is a subclass of DualInputOperator. The solution set is considered the first input, the workset is considered the second input.

Most used methods

getAggregators
getNextWorkset
Gets the contract that has been set as the next workset.
getSolutionSet
Gets the contract that represents the solution set for the step function.
getSolutionSetDelta
Gets the contract that has been set as the solution set delta.
getWorkset
Gets the contract that represents the workset for the step function.
getMaximumNumberOfIterations
getSolutionSetKeyFields
getOperatorInfo
isSolutionSetUnManaged
gets whether the solution set is in managed or unmanaged memory.
getName
<init>
getFirstInput

Popular in Java

Finding current android device location
requestLocationUpdates (LocationManager)
getContentResolver (Context)
getOriginalFilename (MultipartFile)
Return the original filename in the client's filesystem.This may contain path information depending
UnknownHostException (java.net)
Thrown when a hostname can not be resolved.
Time (java.sql)
Java representation of an SQL TIME value. Provides utilities to format and parse the time's represen
NumberFormat (java.text)
The abstract base class for all number formats. This class provides the interface for formatting and
SortedSet (java.util)
SortedSet is a Set which iterates over its elements in a sorted order. The order is determined eithe
HttpServletRequest (javax.servlet.http)
Extends the javax.servlet.ServletRequest interface to provide request information for HTTP servlets.
BasicDataSource (org.apache.commons.dbcp)
Basic implementation of javax.sql.DataSource that is configured via JavaBeans properties. This is no
Top plugins for Android Studio

How to useDeltaIterationBase in org.apache.flink.api.common.operators.base

Best Java code snippets using org.apache.flink.api.common.operators.base.DeltaIterationBase (Showing top 20 results out of 315)

How to use
DeltaIterationBase
in
org.apache.flink.api.common.operators.base