/** * The private constructor that only gets invoked from the Builder. * @param builder */ protected CoGroupOperator(Builder builder) { super(builder.udf, OperatorInfoHelper.binary(), builder.getKeyColumnsArray1(), builder.getKeyColumnsArray2(), builder.name); this.keyTypes = builder.getKeyClassesArray(); if (builder.inputs1 != null && !builder.inputs1.isEmpty()) { setFirstInput(Operator.createUnionCascade(builder.inputs1)); } if (builder.inputs2 != null && !builder.inputs2.isEmpty()) { setSecondInput(Operator.createUnionCascade(builder.inputs2)); } setBroadcastVariables(builder.broadcastInputs); setGroupOrderForInputOne(builder.secondaryOrder1); setGroupOrderForInputTwo(builder.secondaryOrder2); setSemanticProperties(FunctionAnnotation.readDualConstantAnnotations(builder.udf)); }
/** * Creates and returns a CoGroupOperator from using the values given * to the builder. * * @return The created operator */ public CoGroupOperator build() { if (keyClasses.size() <= 0) { throw new IllegalStateException("At least one key attribute has to be set."); } if (name == null) { name = udf.getUserCodeClass().getName(); } return new CoGroupOperator(this); } }
@Override protected Plan getTestJob() { FileDataSource input_left = new FileDataSource(new CoGroupTestInFormat(), leftInPath); DelimitedInputFormat.configureDelimitedFormat(input_left) .recordDelimiter('\n'); input_left.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1)); FileDataSource input_right = new FileDataSource(new CoGroupTestInFormat(), rightInPath); DelimitedInputFormat.configureDelimitedFormat(input_right) .recordDelimiter('\n'); input_right.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1)); CoGroupOperator testCoGrouper = CoGroupOperator.builder(new TestCoGrouper(), StringValue.class, 0, 0) .build(); testCoGrouper.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1)); testCoGrouper.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY, config.getString("CoGroupTest#LocalStrategy", "")); testCoGrouper.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY, config.getString("CoGroupTest#ShipStrategy", "")); FileDataSink output = new FileDataSink(new CoGroupOutFormat(), resultPath); output.setDegreeOfParallelism(1); output.setInput(testCoGrouper); testCoGrouper.setFirstInput(input_left); testCoGrouper.setSecondInput(input_right); return new Plan(output); }
this.messager = CoGroupOperator.builder(MessagingDriver.class, vertexKey, 0, 0) .input2(iteration.getWorkset()) .name("Message Sender") .build(); this.vertexUpdater = CoGroupOperator.builder(VertexUpdateDriver.class, vertexKey, 0, 0) .input1(messager) .input2(iteration.getSolutionSet()) Configuration vertexUdfParams = vertexUpdater.getParameters(); InstantiationUtil.writeObjectToConfig(uf, vertexUdfParams, VertexUpdateDriver.UDF_PARAM); vertexUdfParams.setClass(VertexUpdateDriver.KEY_PARAM, vertexKey); vertexUdfParams.setClass(VertexUpdateDriver.MESSAGE_PARAM, messageType); Configuration messageUdfParams = messager.getParameters(); InstantiationUtil.writeObjectToConfig(mf, messageUdfParams, MessagingDriver.UDF_PARAM); messageUdfParams.setClass(MessagingDriver.KEY_PARAM, vertexKey);
public void setEdgesInput(Operator<Record> c) { this.messager.setFirstInput(c); }
.build(); CoGroupOperator minAndUpdate = CoGroupOperator.builder(new MinIdAndUpdate(), LongValue.class, 0, 0) .input1(joinWithNeighbors) .input2(iteration.getSolutionSet())
CoGroupOperator.builder(new FindShortestPath(), StringValue.class, 0, 0) .keyField(StringValue.class, 1, 1) .name("Find Shortest Paths") .build(); findShortestPaths.setDegreeOfParallelism(numSubTasks); findShortestPaths.setFirstInput(pathsInput); findShortestPaths.setSecondInput(concatPaths); concatPaths.setFirstInput(pathsInput); concatPaths.setSecondInput(pathsInput);
.build(); CoGroupOperator rankAggregation = CoGroupOperator.builder(new DotProductCoGroup(), LongValue.class, 0, 0) .input1(iteration.getPartialSolution()) .input2(join) .name("Rank Aggregation") .build(); rankAggregation.getParameters().setLong(DotProductCoGroup.NUM_VERTICES_PARAMETER, numVertices); rankAggregation.getParameters().setLong(DotProductCoGroup.NUM_DANGLING_VERTICES_PARAMETER, numDanglingVertices);
.build(); CoGroupOperator minAndUpdate = CoGroupOperator.builder(new MinIdAndUpdate(), LongValue.class, 0, 0) .input1(joinWithNeighbors) .input2(iteration.getSolutionSet())
CoGroupOperator antiJoinVisits = CoGroupOperator.builder(new AntiJoinVisits(), StringValue.class, 0, 0) .input1(joinDocsRanks) .input2(filterVisits)