private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) { Ordering groupOrder1 = null; Ordering groupOrder2 = null; CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator(); groupOrder1 = cgc.getGroupOrderForInputOne(); groupOrder2 = cgc.getGroupOrderForInputTwo(); if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) { groupOrder1 = null; } if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) { groupOrder2 = null; } CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2); if (customPartitioner != null) { descr.setCustomPartitioner(customPartitioner); } return Collections.<OperatorDescriptorDual>singletonList(descr); } }
public CoGroupNode(CoGroupOperatorBase<?, ?, ?, ?> operator) { super(operator); this.dataProperties = initializeDataProperties(operator.getCustomPartitioner()); }
getResultType(), name, input1, input2); po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); getInput1Type(), getResultType(), name, input1, input2); po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); new CoGroupOperatorBase<>( function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name); op.setFirstInput(input1); op.setSecondInput(input2); po = op; po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); o.appendOrdering(entry.getLeft(), null, entry.getRight()); po.setGroupOrderForInputOne(o); o.appendOrdering(entry.getLeft(), null, entry.getRight()); po.setGroupOrderForInputTwo(o);
TypeInformation<IN1> inputType1 = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> inputType2 = getOperatorInfo().getSecondInputType(); int[] inputKeys1 = getKeyColumns(0); int[] inputKeys2 = getKeyColumns(1); final TypeSerializer<IN2> inputSerializer2 = inputType2.createSerializer(executionConfig); final TypeComparator<IN1> inputComparator1 = getTypeComparator(executionConfig, inputType1, inputKeys1, inputDirections1); final TypeComparator<IN2> inputComparator2 = getTypeComparator(executionConfig, inputType2, inputKeys2, inputDirections2); System.arraycopy(groupSortDirections, 0, allSortDirections, inputKeys1.length, groupSortDirections.length); inputSortComparator1 = getTypeComparator(executionConfig, inputType1, allSortKeys, allSortDirections); System.arraycopy(groupSortDirections, 0, allSortDirections, inputKeys2.length, groupSortDirections.length); inputSortComparator2 = getTypeComparator(executionConfig, inputType2, allSortKeys, allSortDirections); Collector<OUT> resultCollector = new CopyingListCollector<OUT>(result, getOperatorInfo().getOutputType().createSerializer(executionConfig));
when(op.getSemanticProperties()).thenReturn(origProps); when(op.getKeyColumns(0)).thenReturn(new int[]{3,2}); when(op.getKeyColumns(1)).thenReturn(new int[]{6,3}); when(op.getParameters()).thenReturn(new Configuration());
/** * Gets the order of elements within a group for the first input. * If no such order has been set, this method returns null. * * @return The group order for the first input. */ public Ordering getGroupOrderForInputOne() { return getGroupOrder(0); }
private CoGroupOperatorBase<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>>> getCoGroupOperator( RichCoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>> udf) { TypeInformation<Tuple2<String, Integer>> tuple2Info = TypeInformation.of(new TypeHint<Tuple2<String, Integer>>(){}); return new CoGroupOperatorBase<>( udf, new BinaryOperatorInformation<>(tuple2Info, tuple2Info, tuple2Info), new int[]{0}, new int[]{0}, "coGroup on Collections" ); } }
TypeInformation<IN1> inputType1 = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> inputType2 = getOperatorInfo().getSecondInputType(); int[] inputKeys1 = getKeyColumns(0); int[] inputKeys2 = getKeyColumns(1); final TypeSerializer<IN2> inputSerializer2 = inputType2.createSerializer(executionConfig); final TypeComparator<IN1> inputComparator1 = getTypeComparator(executionConfig, inputType1, inputKeys1, inputDirections1); final TypeComparator<IN2> inputComparator2 = getTypeComparator(executionConfig, inputType2, inputKeys2, inputDirections2); System.arraycopy(groupSortDirections, 0, allSortDirections, inputKeys1.length, groupSortDirections.length); inputSortComparator1 = getTypeComparator(executionConfig, inputType1, allSortKeys, allSortDirections); System.arraycopy(groupSortDirections, 0, allSortDirections, inputKeys2.length, groupSortDirections.length); inputSortComparator2 = getTypeComparator(executionConfig, inputType2, allSortKeys, allSortDirections); Collector<OUT> resultCollector = new CopyingListCollector<OUT>(result, getOperatorInfo().getOutputType().createSerializer(executionConfig));
/** * Gets the order of elements within a group for the second input. * If no such order has been set, this method returns null. * * @return The group order for the second input. */ public Ordering getGroupOrderForInputTwo() { return getGroupOrder(1); }
CoGroupOperatorBase<?, ?, ?, ?> coGroup = (CoGroupOperatorBase<?, ?, ?, ?>) sink.getInput(); assertNotNull(coGroup.getGroupOrderForInputOne()); assertNotNull(coGroup.getGroupOrderForInputTwo()); assertEquals(1, coGroup.getGroupOrderForInputOne().getNumberOfFields()); assertEquals(0, coGroup.getGroupOrderForInputOne().getFieldNumber(0).intValue()); assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputOne().getOrder(0)); assertEquals(2, coGroup.getGroupOrderForInputTwo().getNumberOfFields()); assertEquals(1, coGroup.getGroupOrderForInputTwo().getFieldNumber(0).intValue()); assertEquals(0, coGroup.getGroupOrderForInputTwo().getFieldNumber(1).intValue()); assertEquals(Order.ASCENDING, coGroup.getGroupOrderForInputTwo().getOrder(0)); assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputTwo().getOrder(1));
getResultType(), name, input1, input2); po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); getInput1Type(), getResultType(), name, input1, input2); po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); new CoGroupOperatorBase<>( function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name); op.setFirstInput(input1); op.setSecondInput(input2); po = op; po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); o.appendOrdering(entry.getLeft(), null, entry.getRight()); po.setGroupOrderForInputOne(o); o.appendOrdering(entry.getLeft(), null, entry.getRight()); po.setGroupOrderForInputTwo(o);
TypeInformation<IN1> inputType1 = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> inputType2 = getOperatorInfo().getSecondInputType(); int[] inputKeys1 = getKeyColumns(0); int[] inputKeys2 = getKeyColumns(1); final TypeSerializer<IN2> inputSerializer2 = inputType2.createSerializer(executionConfig); final TypeComparator<IN1> inputComparator1 = getTypeComparator(executionConfig, inputType1, inputKeys1, inputDirections1); final TypeComparator<IN2> inputComparator2 = getTypeComparator(executionConfig, inputType2, inputKeys2, inputDirections2); System.arraycopy(groupSortDirections, 0, allSortDirections, inputKeys1.length, groupSortDirections.length); inputSortComparator1 = getTypeComparator(executionConfig, inputType1, allSortKeys, allSortDirections); System.arraycopy(groupSortDirections, 0, allSortDirections, inputKeys2.length, groupSortDirections.length); inputSortComparator2 = getTypeComparator(executionConfig, inputType2, allSortKeys, allSortDirections); Collector<OUT> resultCollector = new CopyingListCollector<OUT>(result, getOperatorInfo().getOutputType().createSerializer(executionConfig));
public CoGroupNode(CoGroupOperatorBase<?, ?, ?, ?> operator) { super(operator); this.dataProperties = initializeDataProperties(operator.getCustomPartitioner()); }
/** * Gets the order of elements within a group for the second input. * If no such order has been set, this method returns null. * * @return The group order for the second input. */ public Ordering getGroupOrderForInputTwo() { return getGroupOrder(1); }
CoGroupOperatorBase<?, ?, ?, ?> coGroup = (CoGroupOperatorBase<?, ?, ?, ?>) sink.getInput(); assertNotNull(coGroup.getGroupOrderForInputOne()); assertNotNull(coGroup.getGroupOrderForInputTwo()); assertEquals(1, coGroup.getGroupOrderForInputOne().getNumberOfFields()); assertEquals(0, coGroup.getGroupOrderForInputOne().getFieldNumber(0).intValue()); assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputOne().getOrder(0)); assertEquals(2, coGroup.getGroupOrderForInputTwo().getNumberOfFields()); assertEquals(2, coGroup.getGroupOrderForInputTwo().getFieldNumber(0).intValue()); assertEquals(0, coGroup.getGroupOrderForInputTwo().getFieldNumber(1).intValue()); assertEquals(Order.ASCENDING, coGroup.getGroupOrderForInputTwo().getOrder(0)); assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputTwo().getOrder(1));
getResultType(), name, input1, input2); po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); getInput1Type(), getResultType(), name, input1, input2); po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); new CoGroupOperatorBase<>( function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name); op.setFirstInput(input1); op.setSecondInput(input2); po = op; po.setParallelism(getParallelism()); po.setCustomPartitioner(customPartitioner); o.appendOrdering(entry.getLeft(), null, entry.getRight()); po.setGroupOrderForInputOne(o); o.appendOrdering(entry.getLeft(), null, entry.getRight()); po.setGroupOrderForInputTwo(o);
public CoGroupNode(CoGroupOperatorBase<?, ?, ?, ?> operator) { super(operator); this.dataProperties = initializeDataProperties(operator.getCustomPartitioner()); }
/** * Gets the order of elements within a group for the first input. * If no such order has been set, this method returns null. * * @return The group order for the first input. */ public Ordering getGroupOrderForInputOne() { return getGroupOrder(0); }
private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) { Ordering groupOrder1 = null; Ordering groupOrder2 = null; CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator(); groupOrder1 = cgc.getGroupOrderForInputOne(); groupOrder2 = cgc.getGroupOrderForInputTwo(); if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) { groupOrder1 = null; } if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) { groupOrder2 = null; } CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2); if (customPartitioner != null) { descr.setCustomPartitioner(customPartitioner); } return Collections.<OperatorDescriptorDual>singletonList(descr); } }
public CoGroupNode(CoGroupOperatorBase<?, ?, ?, ?> operator) { super(operator); this.dataProperties = initializeDataProperties(operator.getCustomPartitioner()); }