/** * Updates the outputs references of a replicate operator to points to the valid parents. * @param replicateToOutputs where the replicate operators are stored with its valid parents. * @param newOutputs the valid parents of replicate operator. */ private void cleanup(Map<Mutable<ILogicalOperator>, BitSet> replicateToOutputs, List<Pair<Mutable<ILogicalOperator>, Boolean>> newOutputs) { replicateToOutputs.forEach((repRef, allOutputs) -> { newOutputs.clear(); // get the indexes that are set in the BitSet allOutputs.stream().forEach(outIndex -> { newOutputs.add(new Pair<>(((AbstractReplicateOperator) repRef.getValue()).getOutputs().get(outIndex), ((AbstractReplicateOperator) repRef.getValue()).getOutputMaterializationFlags()[outIndex])); }); ((AbstractReplicateOperator) repRef.getValue()).setOutputs(newOutputs); }); }
@Override public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException { return createPropagatingAllInputsTypeEnvironment(ctx); }
@Override public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) { int[] inputDependencyLabels = new int[] { 0 }; AbstractReplicateOperator rop = (AbstractReplicateOperator) op; int[] outputDependencyLabels = new int[rop.getOutputArity()]; // change the labels of outputs that requires materialization to 1 boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags(); for (int i = 0; i < rop.getOutputArity(); i++) { if (outputMaterializationFlags[i]) { outputDependencyLabels[i] = 1; } } return new Pair<>(inputDependencyLabels, outputDependencyLabels); }
ILogicalOperator replicateOutput; sourceNode = destinationNode; for (int i = 0; i < replicateOperator.getOutputs().size(); i++) { replicateOutput = replicateOperator.getOutputs().get(i).getValue(); destinationNodeLabel = formatStringOf(replicateOutput, showDetails); destinationNode = dotBuilder.createNode( DotFormatBuilder.StringValue.of(Integer.toString(replicateOutput.hashCode())), destinationNodeLabel); if (replicateOperator.getOutputMaterializationFlags()[i]) { dotBuilder.createEdge(sourceNode, destinationNode).setColor(DotFormatBuilder.Color.RED); } else {
/** * Collects all replicate operator starting from {@param parent} and all its descendants and keeps track of the * valid parents of a replicate operator. The indexes of valid parents will be set in the BitSet. * @param parent the current operator in consideration for which we want to find replicate op children. * @param replicateToOutputs where the replicate operators will be stored with all its parents (valid & invalid). */ private void findReplicateOp(Mutable<ILogicalOperator> parent, Map<Mutable<ILogicalOperator>, BitSet> replicateToOutputs) { List<Mutable<ILogicalOperator>> children = parent.getValue().getInputs(); for (Mutable<ILogicalOperator> childRef : children) { AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue(); if (child.getOperatorTag() == LogicalOperatorTag.REPLICATE || child.getOperatorTag() == LogicalOperatorTag.SPLIT) { AbstractReplicateOperator replicateChild = (AbstractReplicateOperator) child; int parentIndex = replicateChild.getOutputs().indexOf(parent); if (parentIndex >= 0) { BitSet replicateValidOutputs = replicateToOutputs.get(childRef); if (replicateValidOutputs == null) { replicateValidOutputs = new BitSet(); replicateToOutputs.put(childRef, replicateValidOutputs); } replicateValidOutputs.set(parentIndex); } } findReplicateOp(childRef, replicateToOutputs); } }
private void reviseEdges(IHyracksJobBuilder builder) { /* * revise the edges for the case of replicate operator */ operatorVisitedToParents.forEach((child, parents) -> { if (parents.size() > 1) { if (child.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE || child.getValue().getOperatorTag() == LogicalOperatorTag.SPLIT) { AbstractReplicateOperator rop = (AbstractReplicateOperator) child.getValue(); if (rop.isBlocker()) { // make the order of the graph edges consistent with the order of rop's outputs List<Mutable<ILogicalOperator>> outputs = rop.getOutputs(); for (Mutable<ILogicalOperator> parent : parents) { builder.contributeGraphEdge(child.getValue(), outputs.indexOf(parent), parent.getValue(), 0); } } else { int i = 0; for (Mutable<ILogicalOperator> parent : parents) { builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0); i++; } } } } }); } }
/** * Checks whether the operator {@code op} is supposed to be materialized * due to a replicate/split operators. * * @param op * @return true if the operator will be materialized. Otherwise false */ private boolean isMaterialized(ILogicalOperator op) { for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) { final ILogicalOperator inputOp = inputOpRef.getValue(); final LogicalOperatorTag inputOpTag = inputOp.getOperatorTag(); if (inputOpTag == LogicalOperatorTag.REPLICATE || inputOpTag == LogicalOperatorTag.SPLIT) { final AbstractReplicateOperator replicateOperator = (AbstractReplicateOperator) inputOp; if (replicateOperator.isMaterialized(op)) { return true; } } } return false; }
&& ((DelegateOperator) op).getDelegate() instanceof CommitOperator)) { for (Map.Entry<AbstractReplicateOperator, MutableInt> entry : replicateOperators.entrySet()) { if (entry.getKey().getOutputs().size() != entry.getValue().getValue()) { throw new CompilationException(ErrorCode.COMPILATION_FAILED_DUE_TO_REPLICATE_OP, op.getSourceLocation()); if (replicateOperator.getOutputs().contains(replicateActualParent)) { if (replicateOperators.get(replicateOperator).getValue() == replicateOperator.getOutputs().size()) { context.addToDontApplySet(this, opRef.getValue()); for (int oldParentIndex = 0; oldParentIndex < replicateOperator.getOutputs().size(); oldParentIndex++) { if (parentsPathToReplicate.contains(replicateOperator.getOutputs().get(oldParentIndex))) { replicateOperator.getOutputs().set(oldParentIndex, replicateActualParent); parentFixed = true; updateNumberOfParentsDone(replicateOperator); if (replicateOperators.get(replicateOperator).getValue() == replicateOperator.getOutputs().size()) { context.addToDontApplySet(this, opRef.getValue());
|| opRef.getValue().getOperatorTag() == LogicalOperatorTag.SPLIT) { AbstractReplicateOperator rop = (AbstractReplicateOperator) opRef.getValue(); List<Mutable<ILogicalOperator>> outputs = rop.getOutputs(); for (outputIndex = 0; outputIndex < outputs.size(); outputIndex++) { if (outputs.get(outputIndex).equals(parentRef)) {