public PartitionNode(PartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new PartitionDescriptor( this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(), operator.getDistribution()); this.possibleProperties = Collections.singletonList(descr); }
@SuppressWarnings("unchecked") private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner( SelectorFunctionKeys<T, ?> rawKeys, PartitionMethod pMethod, String name, Operator<T> input, int partitionDop, Partitioner<?> customPartitioner, Order[] orders) { final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys; TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput = new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name); keyedPartitionedInput.setInput(keyedInput); keyedPartitionedInput.setCustomPartitioner(customPartitioner); keyedPartitionedInput.setParallelism(partitionDop); keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING)); return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys); }
public void setCustomPartitioner(Partitioner<?> customPartitioner) { if (customPartitioner != null) { int[] keys = getKeyColumns(0); if (keys == null || keys.length == 0) { throw new IllegalArgumentException("Cannot use custom partitioner for a non-grouped GroupReduce (AllGroupReduce)"); } if (keys.length > 1) { throw new IllegalArgumentException("Cannot use the key partitioner for composite keys (more than one key field)"); } } this.customPartitioner = customPartitioner; }
public PartitionNode(PartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new PartitionDescriptor( this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(), operator.getDistribution()); this.possibleProperties = Collections.singletonList(descr); }
PartitionOperatorBase<T> rebalancedInput = new PartitionOperatorBase<>(operatorInfo, pMethod, name); rebalancedInput.setInput(input); rebalancedInput.setParallelism(getParallelism()); PartitionOperatorBase<T> partitionedInput = new PartitionOperatorBase<>(operatorInfo, pMethod, logicalKeyPositions, name); partitionedInput.setInput(input); partitionedInput.setParallelism(getParallelism()); partitionedInput.setDistribution(distribution); partitionedInput.setCustomPartitioner(customPartitioner); partitionedInput.setOrdering(computeOrdering(pKeys, orders));
public void setCustomPartitioner(Partitioner<?> customPartitioner) { if (customPartitioner != null) { int[] keys = getKeyColumns(0); if (keys == null || keys.length == 0) { throw new IllegalArgumentException("Cannot use custom partitioner for a non-grouped GroupReduce (AllGroupReduce)"); } if (keys.length > 1) { throw new IllegalArgumentException("Cannot use the key partitioner for composite keys (more than one key field)"); } } this.customPartitioner = customPartitioner; }
public PartitionNode(PartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new PartitionDescriptor( this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(), operator.getDistribution()); this.possibleProperties = Collections.singletonList(descr); }
@SuppressWarnings("unchecked") private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner( SelectorFunctionKeys<T, ?> rawKeys, PartitionMethod pMethod, String name, Operator<T> input, int partitionDop, Partitioner<?> customPartitioner, Order[] orders) { final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys; TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput = new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name); keyedPartitionedInput.setInput(keyedInput); keyedPartitionedInput.setCustomPartitioner(customPartitioner); keyedPartitionedInput.setParallelism(partitionDop); keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING)); return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys); }
public void setCustomPartitioner(Partitioner<?> customPartitioner) { if (customPartitioner != null) { int[] keys = getKeyColumns(0); if (keys == null || keys.length == 0) { throw new IllegalArgumentException("Cannot use custom partitioner for a non-grouped GroupReduce (AllGroupReduce)"); } if (keys.length > 1) { throw new IllegalArgumentException("Cannot use the key partitioner for composite keys (more than one key field)"); } } this.customPartitioner = customPartitioner; }
public PartitionNode(PartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new PartitionDescriptor( this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(), operator.getDistribution()); this.possibleProperties = Collections.singletonList(descr); }
@SuppressWarnings("unchecked") private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner( SelectorFunctionKeys<T, ?> rawKeys, PartitionMethod pMethod, String name, Operator<T> input, int partitionDop, Partitioner<?> customPartitioner, Order[] orders) { final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys; TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput = new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name); keyedPartitionedInput.setInput(keyedInput); keyedPartitionedInput.setCustomPartitioner(customPartitioner); keyedPartitionedInput.setParallelism(partitionDop); keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING)); return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys); }
public PartitionNode(PartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new PartitionDescriptor( this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(), operator.getDistribution()); this.possibleProperties = Collections.singletonList(descr); }
PartitionOperatorBase<T> rebalancedInput = new PartitionOperatorBase<>(operatorInfo, pMethod, name); rebalancedInput.setInput(input); rebalancedInput.setParallelism(getParallelism()); PartitionOperatorBase<T> partitionedInput = new PartitionOperatorBase<>(operatorInfo, pMethod, logicalKeyPositions, name); partitionedInput.setInput(input); partitionedInput.setParallelism(getParallelism()); partitionedInput.setDistribution(distribution); partitionedInput.setCustomPartitioner(customPartitioner); partitionedInput.setOrdering(computeOrdering(pKeys, orders));
PartitionOperatorBase<T> rebalancedInput = new PartitionOperatorBase<>(operatorInfo, pMethod, name); rebalancedInput.setInput(input); rebalancedInput.setParallelism(getParallelism()); PartitionOperatorBase<T> partitionedInput = new PartitionOperatorBase<>(operatorInfo, pMethod, logicalKeyPositions, name); partitionedInput.setInput(input); partitionedInput.setParallelism(getParallelism()); partitionedInput.setDistribution(distribution); partitionedInput.setCustomPartitioner(customPartitioner); partitionedInput.setOrdering(computeOrdering(pKeys, orders));