private List<OperatorDescriptorDual> getDataProperties() { OuterJoinOperatorBase<?, ?, ?, ?> operator = getOperator(); OuterJoinType type = operator.getOuterJoinType(); JoinHint joinHint = operator.getJoinHint(); joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint; List<OperatorDescriptorDual> list; switch (type) { case LEFT: list = createLeftOuterJoinDescriptors(joinHint); break; case RIGHT: list = createRightOuterJoinDescriptors(joinHint); break; case FULL: list = createFullOuterJoinDescriptors(joinHint); break; default: throw new CompilerException("Unknown outer join type: " + type); } Partitioner<?> customPartitioner = operator.getCustomPartitioner(); if (customPartitioner != null) { for (OperatorDescriptorDual desc : list) { ((AbstractJoinDescriptor) desc).setCustomPartitioner(customPartitioner); } } return list; }
@SuppressWarnings({"rawtypes", "unchecked"}) @Before public void setup() { joiner = new MockRichFlatJoinFunction(); baseOperator = new OuterJoinOperatorBase(joiner, new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null); executionConfig = new ExecutionConfig(); String taskName = "Test rich outer join function"; TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0); HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>(); HashMap<String, Future<Path>> cpTasks = new HashMap<>(); runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()); }
@Override protected List<OUT> executeOnCollections(List<IN1> leftInput, List<IN2> rightInput, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception { TypeInformation<IN1> leftInformation = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> rightInformation = getOperatorInfo().getSecondInputType(); TypeInformation<OUT> outInformation = getOperatorInfo().getOutputType(); TypeComparator<IN1> leftComparator = buildComparatorFor(0, executionConfig, leftInformation); TypeComparator<IN2> rightComparator = buildComparatorFor(1, executionConfig, rightInformation); TypeSerializer<IN1> leftSerializer = leftInformation.createSerializer(executionConfig); TypeSerializer<IN2> rightSerializer = rightInformation.createSerializer(executionConfig); OuterJoinListIterator<IN1, IN2> outerJoinIterator = new OuterJoinListIterator<>(leftInput, leftSerializer, leftComparator, rightInput, rightSerializer, rightComparator, outerJoinType); // -------------------------------------------------------------------- // Run UDF // -------------------------------------------------------------------- FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, runtimeContext); FunctionUtils.openFunction(function, this.parameters); List<OUT> result = new ArrayList<>(); Collector<OUT> collector = new CopyingListCollector<>(result, outInformation.createSerializer(executionConfig)); while (outerJoinIterator.next()) { IN1 left = outerJoinIterator.getLeft(); IN2 right = outerJoinIterator.getRight(); function.join(left == null ? null : leftSerializer.copy(left), right == null ? null : rightSerializer.copy(right), collector); } FunctionUtils.closeFunction(function); return result; }
@SuppressWarnings("unchecked") private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) { TypeComparator<T> comparator; if (typeInformation instanceof AtomicType) { comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig); } else if (typeInformation instanceof CompositeType) { int[] keyPositions = getKeyColumns(input); boolean[] orders = new boolean[keyPositions.length]; Arrays.fill(orders, true); comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig); } else { throw new RuntimeException("Type information for input of type " + typeInformation.getClass() .getCanonicalName() + " is not supported. Could not generate a comparator."); } return comparator; }
@SuppressWarnings("unchecked") private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) { TypeComparator<T> comparator; if (typeInformation instanceof AtomicType) { comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig); } else if (typeInformation instanceof CompositeType) { int[] keyPositions = getKeyColumns(input); boolean[] orders = new boolean[keyPositions.length]; Arrays.fill(orders, true); comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig); } else { throw new RuntimeException("Type information for input of type " + typeInformation.getClass() .getCanonicalName() + " is not supported. Could not generate a comparator."); } return comparator; }
private List<OperatorDescriptorDual> getDataProperties() { OuterJoinOperatorBase<?, ?, ?, ?> operator = getOperator(); OuterJoinType type = operator.getOuterJoinType(); JoinHint joinHint = operator.getJoinHint(); joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint; List<OperatorDescriptorDual> list; switch (type) { case LEFT: list = createLeftOuterJoinDescriptors(joinHint); break; case RIGHT: list = createRightOuterJoinDescriptors(joinHint); break; case FULL: list = createFullOuterJoinDescriptors(joinHint); break; default: throw new CompilerException("Unknown outer join type: " + type); } Partitioner<?> customPartitioner = operator.getCustomPartitioner(); if (customPartitioner != null) { for (OperatorDescriptorDual desc : list) { ((AbstractJoinDescriptor) desc).setCustomPartitioner(customPartitioner); } } return list; }
@Override protected List<OUT> executeOnCollections(List<IN1> leftInput, List<IN2> rightInput, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception { TypeInformation<IN1> leftInformation = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> rightInformation = getOperatorInfo().getSecondInputType(); TypeInformation<OUT> outInformation = getOperatorInfo().getOutputType(); TypeComparator<IN1> leftComparator = buildComparatorFor(0, executionConfig, leftInformation); TypeComparator<IN2> rightComparator = buildComparatorFor(1, executionConfig, rightInformation); TypeSerializer<IN1> leftSerializer = leftInformation.createSerializer(executionConfig); TypeSerializer<IN2> rightSerializer = rightInformation.createSerializer(executionConfig); OuterJoinListIterator<IN1, IN2> outerJoinIterator = new OuterJoinListIterator<>(leftInput, leftSerializer, leftComparator, rightInput, rightSerializer, rightComparator, outerJoinType); // -------------------------------------------------------------------- // Run UDF // -------------------------------------------------------------------- FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, runtimeContext); FunctionUtils.openFunction(function, this.parameters); List<OUT> result = new ArrayList<>(); Collector<OUT> collector = new CopyingListCollector<>(result, outInformation.createSerializer(executionConfig)); while (outerJoinIterator.next()) { IN1 left = outerJoinIterator.getLeft(); IN2 right = outerJoinIterator.getRight(); function.join(left == null ? null : leftSerializer.copy(left), right == null ? null : rightSerializer.copy(right), collector); } FunctionUtils.closeFunction(function); return result; }
@SuppressWarnings("unchecked") public JoinOperatorBase<?, ?, OUT, ?> build() { JoinOperatorBase<?, ?, OUT, ?> operator; if (joinType.isOuter()) { operator = new OuterJoinOperatorBase<>( udf, new BinaryOperatorInformation(input1Type, input2Type, resultType), this.keys1.computeLogicalKeyPositions(), this.keys2.computeLogicalKeyPositions(), this.name, getOuterJoinType()); } else { operator = new InnerJoinOperatorBase<>( udf, new BinaryOperatorInformation(input1Type, input2Type, resultType), this.keys1.computeLogicalKeyPositions(), this.keys2.computeLogicalKeyPositions(), this.name); } operator.setFirstInput(input1); operator.setSecondInput(input2); operator.setParallelism(parallelism); operator.setCustomPartitioner(partitioner); operator.setJoinHint(joinHint); return operator; }
@SuppressWarnings("unchecked") private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) { TypeComparator<T> comparator; if (typeInformation instanceof AtomicType) { comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig); } else if (typeInformation instanceof CompositeType) { int[] keyPositions = getKeyColumns(input); boolean[] orders = new boolean[keyPositions.length]; Arrays.fill(orders, true); comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig); } else { throw new RuntimeException("Type information for input of type " + typeInformation.getClass() .getCanonicalName() + " is not supported. Could not generate a comparator."); } return comparator; }
private List<OperatorDescriptorDual> getDataProperties() { OuterJoinOperatorBase<?, ?, ?, ?> operator = getOperator(); OuterJoinType type = operator.getOuterJoinType(); JoinHint joinHint = operator.getJoinHint(); joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint; List<OperatorDescriptorDual> list; switch (type) { case LEFT: list = createLeftOuterJoinDescriptors(joinHint); break; case RIGHT: list = createRightOuterJoinDescriptors(joinHint); break; case FULL: list = createFullOuterJoinDescriptors(joinHint); break; default: throw new CompilerException("Unknown outer join type: " + type); } Partitioner<?> customPartitioner = operator.getCustomPartitioner(); if (customPartitioner != null) { for (OperatorDescriptorDual desc : list) { ((AbstractJoinDescriptor) desc).setCustomPartitioner(customPartitioner); } } return list; }
@Override protected List<OUT> executeOnCollections(List<IN1> leftInput, List<IN2> rightInput, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception { TypeInformation<IN1> leftInformation = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> rightInformation = getOperatorInfo().getSecondInputType(); TypeInformation<OUT> outInformation = getOperatorInfo().getOutputType(); TypeComparator<IN1> leftComparator = buildComparatorFor(0, executionConfig, leftInformation); TypeComparator<IN2> rightComparator = buildComparatorFor(1, executionConfig, rightInformation); TypeSerializer<IN1> leftSerializer = leftInformation.createSerializer(executionConfig); TypeSerializer<IN2> rightSerializer = rightInformation.createSerializer(executionConfig); OuterJoinListIterator<IN1, IN2> outerJoinIterator = new OuterJoinListIterator<>(leftInput, leftSerializer, leftComparator, rightInput, rightSerializer, rightComparator, outerJoinType); // -------------------------------------------------------------------- // Run UDF // -------------------------------------------------------------------- FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, runtimeContext); FunctionUtils.openFunction(function, this.parameters); List<OUT> result = new ArrayList<>(); Collector<OUT> collector = new CopyingListCollector<>(result, outInformation.createSerializer(executionConfig)); while (outerJoinIterator.next()) { IN1 left = outerJoinIterator.getLeft(); IN2 right = outerJoinIterator.getRight(); function.join(left == null ? null : leftSerializer.copy(left), right == null ? null : rightSerializer.copy(right), collector); } FunctionUtils.closeFunction(function); return result; }
@SuppressWarnings("unchecked") public JoinOperatorBase<?, ?, OUT, ?> build() { JoinOperatorBase<?, ?, OUT, ?> operator; if (joinType.isOuter()) { operator = new OuterJoinOperatorBase<>( udf, new BinaryOperatorInformation(input1Type, input2Type, resultType), this.keys1.computeLogicalKeyPositions(), this.keys2.computeLogicalKeyPositions(), this.name, getOuterJoinType()); } else { operator = new InnerJoinOperatorBase<>( udf, new BinaryOperatorInformation(input1Type, input2Type, resultType), this.keys1.computeLogicalKeyPositions(), this.keys2.computeLogicalKeyPositions(), this.name); } operator.setFirstInput(input1); operator.setSecondInput(input2); operator.setParallelism(parallelism); operator.setCustomPartitioner(partitioner); operator.setJoinHint(joinHint); return operator; }
private List<OperatorDescriptorDual> getDataProperties() { OuterJoinOperatorBase<?, ?, ?, ?> operator = getOperator(); OuterJoinType type = operator.getOuterJoinType(); JoinHint joinHint = operator.getJoinHint(); joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint; List<OperatorDescriptorDual> list; switch (type) { case LEFT: list = createLeftOuterJoinDescriptors(joinHint); break; case RIGHT: list = createRightOuterJoinDescriptors(joinHint); break; case FULL: list = createFullOuterJoinDescriptors(joinHint); break; default: throw new CompilerException("Unknown outer join type: " + type); } Partitioner<?> customPartitioner = operator.getCustomPartitioner(); if (customPartitioner != null) { for (OperatorDescriptorDual desc : list) { ((AbstractJoinDescriptor) desc).setCustomPartitioner(customPartitioner); } } return list; }
@SuppressWarnings("unchecked") public JoinOperatorBase<?, ?, OUT, ?> build() { JoinOperatorBase<?, ?, OUT, ?> operator; if (joinType.isOuter()) { operator = new OuterJoinOperatorBase<>( udf, new BinaryOperatorInformation(input1Type, input2Type, resultType), this.keys1.computeLogicalKeyPositions(), this.keys2.computeLogicalKeyPositions(), this.name, getOuterJoinType()); } else { operator = new InnerJoinOperatorBase<>( udf, new BinaryOperatorInformation(input1Type, input2Type, resultType), this.keys1.computeLogicalKeyPositions(), this.keys2.computeLogicalKeyPositions(), this.name); } operator.setFirstInput(input1); operator.setSecondInput(input2); operator.setParallelism(parallelism); operator.setCustomPartitioner(partitioner); operator.setJoinHint(joinHint); return operator; }
private List<OperatorDescriptorDual> getDataProperties() { OuterJoinOperatorBase<?, ?, ?, ?> operator = getOperator(); OuterJoinType type = operator.getOuterJoinType(); JoinHint joinHint = operator.getJoinHint(); joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint; List<OperatorDescriptorDual> list; switch (type) { case LEFT: list = createLeftOuterJoinDescriptors(joinHint); break; case RIGHT: list = createRightOuterJoinDescriptors(joinHint); break; case FULL: list = createFullOuterJoinDescriptors(joinHint); break; default: throw new CompilerException("Unknown outer join type: " + type); } Partitioner<?> customPartitioner = operator.getCustomPartitioner(); if (customPartitioner != null) { for (OperatorDescriptorDual desc : list) { ((AbstractJoinDescriptor) desc).setCustomPartitioner(customPartitioner); } } return list; }