/** * Checks, whether a combiner function has been given for the function encapsulated * by this reduce contract. * * @return True, if a combiner has been given, false otherwise. */ public boolean isCombineable() { return getPactContract().isCombinable(); }
new GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>>(function, operatorInfo, new int[0], name); po.setCombinable(true); po.setInput(input); po.setDegreeOfParallelism(this.getParallelism()); UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>> po = new GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(true); po.setInput(input); po.setDegreeOfParallelism(this.getParallelism()); po.setSemanticProperties(props);
new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, new int[0], name); po.setCombinable(combinable); po.setInput(input); po.setDegreeOfParallelism(1); return po; UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po = new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(combinable); po.setInput(input); po.setDegreeOfParallelism(this.getParallelism()); o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]); po.setGroupOrder(o);
@Override protected List<OperatorDescriptorSingle> getPossibleProperties() { final Configuration conf = getPactContract().getParameters(); final String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null); } else if (PactCompiler.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) { if (!isCombineable()) { PactCompiler.LOG.warn("Strategy hint for Reduce Pact '" + getPactContract().getName() + "' desires combinable reduce, but user function is not marked combinable."); groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) getPactContract()).getGroupOrder(); if (groupOrder != null && groupOrder.getNumberOfFields() == 0) { groupOrder = null;
@Override protected void getSingleInputNodeSchema(SingleInputPlanNode node, SparseKeySchema schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException { // check that we got the right types SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getPactContract(); if (! (contract instanceof RecordOperator)) { throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation."); } RecordOperator recContract = (RecordOperator) contract; // add the information to the schema int[] localPositions = contract.getKeyColumns(0); Class<? extends Key<?>>[] types = recContract.getKeyClasses(); for (int i = 0; i < localPositions.length; i++) { schema.addType(localPositions[i], types[i]); } // this is a temporary fix, we should solve this more generic if (contract instanceof GroupReduceOperatorBase) { Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder(); if (groupOrder != null) { addOrderingToSchema(groupOrder, schema); } } }
(source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_COMBINE)) { GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType();
public PlanUnwrappingReduceGroupOperator(GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name, TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey, boolean combinable) { super(combinable ? new TupleUnwrappingCombinableGroupReducer<IN, OUT, K>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf), new UnaryOperatorInformation<Tuple2<K, IN>, OUT>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
@Override protected List<OperatorDescriptorSingle> getPossibleProperties() { final Configuration conf = getPactContract().getParameters(); final String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null); } else if (PactCompiler.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) { if (!isCombineable()) { PactCompiler.LOG.warn("Strategy hint for Reduce Pact '" + getPactContract().getName() + "' desires combinable reduce, but user function is not marked combinable."); groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) getPactContract()).getGroupOrder(); if (groupOrder != null && groupOrder.getNumberOfFields() == 0) { groupOrder = null;
@Override protected void getSingleInputNodeSchema(SingleInputPlanNode node, SparseKeySchema schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException { // check that we got the right types SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getPactContract(); if (! (contract instanceof RecordOperator)) { throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation."); } RecordOperator recContract = (RecordOperator) contract; // add the information to the schema int[] localPositions = contract.getKeyColumns(0); Class<? extends Key<?>>[] types = recContract.getKeyClasses(); for (int i = 0; i < localPositions.length; i++) { schema.addType(localPositions[i], types[i]); } // this is a temporary fix, we should solve this more generic if (contract instanceof GroupReduceOperatorBase) { Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder(); if (groupOrder != null) { addOrderingToSchema(groupOrder, schema); } } }
(source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_COMBINE)) { GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType();
/** * Checks, whether a combiner function has been given for the function encapsulated * by this reduce contract. * * @return True, if a combiner has been given, false otherwise. */ public boolean isCombineable() { return getPactContract().isCombinable(); }
@Override public boolean isCombinable() { return super.isCombinable() || getUserCodeWrapper().getUserCodeAnnotation(Combinable.class) != null; }