new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, new int[0], name); po.setCombinable(true); po.setInput(input); po.setParallelism(this.getParallelism()); UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> po = new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(true); po.setInput(input); po.setParallelism(this.getParallelism()); po.setSemanticProperties(props); po.setCustomPartitioner(grouping.getCustomPartitioner());
assertEquals(1, reducer.getKeyColumns(0).length); assertEquals(0, reducer.getKeyColumns(0)[0]); assertEquals(-1, reducer.getParallelism()); assertTrue(reducer.isCombinable()); assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) { final Configuration conf = getOperator().getParameters(); final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null); Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() + "' requires combinable reduce, but user function is not marked combinable."); groupOrder = getOperator().getGroupOrder(); if (groupOrder != null && groupOrder.getNumberOfFields() == 0) { groupOrder = null;
new GroupReduceOperatorBase<>(function, operatorInfo, new int[0], name); po.setCombinable(combinable); po.setInput(input); po.setParallelism(1); return po; UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po = new GroupReduceOperatorBase<>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(combinable); po.setInput(input); po.setParallelism(getParallelism()); po.setCustomPartitioner(grouper.getCustomPartitioner()); o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]); po.setGroupOrder(o);
final GroupReduceOperatorBase sicOperatorBase = new GroupReduceOperatorBase(sampleInCoordinator, sicOperatorInformation, SIC_NAME); final GroupReduceNode sicNode = new GroupReduceNode(sicOperatorBase); final Channel sicChannel = new Channel(sipPlanNode, TempMode.NONE);
new GroupReduceOperatorBase<>( reducer, new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE), List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, null, executionConfig); executionConfig.enableObjectReuse(); List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, null, executionConfig);
when(op.getSemanticProperties()).thenReturn(origProps); when(op.getKeyColumns(0)).thenReturn(new int[]{3,2}); when(op.getParameters()).thenReturn(new Configuration());
public PlanUnwrappingReduceGroupOperator( GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name, TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey, boolean combinable) { super( combinable ? new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf), new UnaryOperatorInformation<>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
/** * Checks, whether a combiner function has been given for the function encapsulated * by this reduce contract. * * @return True, if a combiner has been given, false otherwise. */ public boolean isCombineable() { return getOperator().isCombinable(); }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for GroupReduce may only be preserved on key fields. SingleInputSemanticProperties origProps = getOperator().getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } // only add forward field information for key fields if(this.keys != null) { for (int f : this.keys) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for (int t : targets) { filteredProps.addForwardedField(f, t); } } } return filteredProps; }
(source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_REDUCE_COMBINE)) { GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType();
new GroupReduceOperatorBase<>(function, operatorInfo, new int[0], name); po.setCombinable(combinable); po.setInput(input); po.setParallelism(1); return po; UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po = new GroupReduceOperatorBase<>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(combinable); po.setInput(input); po.setParallelism(getParallelism()); po.setCustomPartitioner(grouper.getCustomPartitioner()); o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]); po.setGroupOrder(o);
final GroupReduceOperatorBase sicOperatorBase = new GroupReduceOperatorBase(sampleInCoordinator, sicOperatorInformation, SIC_NAME); final GroupReduceNode sicNode = new GroupReduceNode(sicOperatorBase); final Channel sicChannel = new Channel(sipPlanNode, TempMode.NONE);
new GroupReduceOperatorBase<>( reducer, new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE), List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<>(), List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<>(),
public PlanUnwrappingSortedReduceGroupOperator( GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K1> groupingKey, Keys.SelectorFunctionKeys<IN, K2> sortingKey, String name, TypeInformation<OUT> outType, TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey, boolean combinable) { super( combinable ? new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K1, K2>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K1, K2>(udf), new UnaryOperatorInformation<>(typeInfoWithKey, outType), groupingKey.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
/** * Checks, whether a combiner function has been given for the function encapsulated * by this reduce contract. * * @return True, if a combiner has been given, false otherwise. */ public boolean isCombineable() { return getOperator().isCombinable(); }
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for GroupReduce may only be preserved on key fields. SingleInputSemanticProperties origProps = getOperator().getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } // only add forward field information for key fields if(this.keys != null) { for (int f : this.keys) { FieldSet targets = origProps.getForwardingTargetFields(0, f); for (int t : targets) { filteredProps.addForwardedField(f, t); } } } return filteredProps; }
(source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_REDUCE_COMBINE)) { GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType();
new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, new int[0], name); po.setCombinable(true); po.setInput(input); po.setParallelism(this.getParallelism()); UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> po = new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(true); po.setInput(input); po.setParallelism(this.getParallelism()); po.setCustomPartitioner(grouping.getCustomPartitioner()); po.setSemanticProperties(props);
new GroupReduceOperatorBase<>(function, operatorInfo, new int[0], name); po.setCombinable(combinable); po.setInput(input); po.setParallelism(1); return po; UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType()); GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po = new GroupReduceOperatorBase<>(function, operatorInfo, logicalKeyPositions, name); po.setCombinable(combinable); po.setInput(input); po.setParallelism(getParallelism()); po.setCustomPartitioner(grouper.getCustomPartitioner()); o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]); po.setGroupOrder(o);