public SortPartitionNode(SortPartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering()); this.possibleProperties = Collections.singletonList(descr); }
SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(input); if (this.getParallelism() < 0) { noop.setParallelism(input.getParallelism()); } else { noop.setParallelism(this.getParallelism());
@Override protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) { TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType(); int[] sortColumns = this.partitionOrdering.getFieldPositions(); boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections(); final TypeComparator<IN> sortComparator; if (inputType instanceof CompositeType) { sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig); } else if (inputType instanceof AtomicType) { sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig); } else { throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet."); } Collections.sort(inputData, new Comparator<IN>() { @Override public int compare(IN o1, IN o2) { return sortComparator.compare(o1, o2); } }); return inputData; } }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }
public SortPartitionNode(SortPartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering()); this.possibleProperties = Collections.singletonList(descr); }
@Override protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) { TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType(); int[] sortColumns = this.partitionOrdering.getFieldPositions(); boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections(); final TypeComparator<IN> sortComparator; if (inputType instanceof CompositeType) { sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig); } else if (inputType instanceof AtomicType) { sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig); } else { throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet."); } Collections.sort(inputData, new Comparator<IN>() { @Override public int compare(IN o1, IN o2) { return sortComparator.compare(o1, o2); } }); return inputData; } }
SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(input); if (this.getParallelism() < 0) { noop.setParallelism(input.getParallelism()); } else { noop.setParallelism(this.getParallelism());
public SortPartitionNode(SortPartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering()); this.possibleProperties = Collections.singletonList(descr); }
@Override protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) { TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType(); int[] sortColumns = this.partitionOrdering.getFieldPositions(); boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections(); final TypeComparator<IN> sortComparator; if (inputType instanceof CompositeType) { sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig); } else if (inputType instanceof AtomicType) { sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig); } else { throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet."); } Collections.sort(inputData, new Comparator<IN>() { @Override public int compare(IN o1, IN o2) { return sortComparator.compare(o1, o2); } }); return inputData; } }
SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(input); if (this.getParallelism() < 0) { noop.setParallelism(input.getParallelism()); } else { noop.setParallelism(this.getParallelism());
public SortPartitionNode(SortPartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering()); this.possibleProperties = Collections.singletonList(descr); }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }
public SortPartitionNode(SortPartitionOperatorBase<?> operator) { super(operator); OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering()); this.possibleProperties = Collections.singletonList(descr); }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }