JobVertex mapSinkVertex = verticesSorted.get(1); assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType()); assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapSinkVertex.getInputs().get(0).getSource().getResultType());
JobVertex predecessor = edge.getSource().getProducer(); gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
result.getId(), this, numTaskVertices, result.getResultType());
public Collection<Collection<ExecutionVertexID>> getResultPartitionConsumerExecutionVertices( IntermediateDataSetID resultID, int partitionNumber) { IntermediateDataSet result = getResult(resultID); if (result == null) { throw new IllegalArgumentException("Cannot find the given result " + resultID + " in job graph"); } if (partitionNumber >= result.getProducer().getParallelism()) { throw new IllegalArgumentException("Result partition index out of bounds: " + partitionNumber + "/" + result.getProducer().getParallelism()); } Collection<Collection<ExecutionVertexID>> consumerVertices = new ArrayList<>(); for (JobEdge edge : getResult(resultID).getConsumers()) { consumerVertices.add(edge.getConsumerExecutionVertices(partitionNumber)); } return consumerVertices; }
JobVertex producerJobVertex = jobGraph.findVertexByID(jobGraph.getResultProducerID(resultID)); for (int i = 0; i < producerJobVertex.getParallelism(); i++) { for (JobEdge edge : jobGraph.getResult(resultID).getConsumers()) { for (ExecutionVertexID vertexID : edge.getConsumerExecutionVertices(i)) { if (!vertexInputsMap.containsKey(vertexID)) { vertexID.getJobVertexID(), resultID, jobGraph.getResult(resultID).getResultType().isPipelined());
/** * Constructs a new job edge, that connects an intermediate result to a consumer task. * * @param source The data set that is at the source of this edge. * @param target The operation that is at the target of this edge. * @param distributionPattern The pattern that defines how the connection behaves in parallel. */ public JobEdge(IntermediateDataSet source, JobVertex target, DistributionPattern distributionPattern) { if (source == null || target == null || distributionPattern == null) { throw new NullPointerException(); } this.target = target; this.distributionPattern = distributionPattern; this.source = source; this.sourceId = source.getId(); }
public JobEdge connectDataSetAsInput(IntermediateDataSet dataSet, DistributionPattern distPattern) { JobEdge edge = new JobEdge(dataSet, this, distPattern); this.inputs.add(edge); dataSet.addConsumer(edge); return edge; }
public JobVertexID getResultProducerID(IntermediateDataSetID resultID) { IntermediateDataSet result = getResult(resultID); if (result == null) { throw new IllegalArgumentException("Cannot find the given result " + resultID + " in job graph"); } return result.getProducer().getID(); }
public IntermediateDataSet createAndAddResultDataSet( IntermediateDataSetID id, ResultPartitionType partitionType) { IntermediateDataSet result = new IntermediateDataSet(id, partitionType, this); this.results.add(result); return result; }
/** * Sets the parallelism for the task. * * @param parallelism The parallelism for the task. */ public void setParallelism(int parallelism) { if (parallelism < 1) { throw new IllegalArgumentException("The parallelism must be at least one."); } this.parallelism = parallelism; // Clear the consumer execution vertices cache for related edges for (JobEdge edge : getInputs()) { edge.clearConsumerExecutionVerticesCache(); } for (IntermediateDataSet dataSet : getProducedDataSets()) { for (JobEdge edge :dataSet.getConsumers()) { edge.clearConsumerExecutionVerticesCache(); } } }
} else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s), %s, %s", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName(), edge.getDistributionPattern(), edge.getSource().getResultType()));
result.getId(), this, numTaskVertices, result.getResultType());
private void addNodesThatHaveNoNewPredecessors(JobVertex start, List<JobVertex> target, Set<JobVertex> remaining) { // forward traverse over all produced data sets and all their consumers for (IntermediateDataSet dataSet : start.getProducedDataSets()) { for (JobEdge edge : dataSet.getConsumers()) { // a vertex can be added, if it has no predecessors that are still in the 'remaining' set JobVertex v = edge.getTarget(); if (!remaining.contains(v)) { continue; } boolean hasNewPredecessors = false; for (JobEdge e : v.getInputs()) { // skip the edge through which we came if (e == edge) { continue; } IntermediateDataSet source = e.getSource(); if (remaining.contains(source.getProducer())) { hasNewPredecessors = true; break; } } if (!hasNewPredecessors) { target.add(v); remaining.remove(v); addNodesThatHaveNoNewPredecessors(v, target, remaining); } } } }
/** * Constructs a new job edge, that connects an intermediate result to a consumer task. * * @param source The data set that is at the source of this edge. * @param target The operation that is at the target of this edge. * @param distributionPattern The pattern that defines how the connection behaves in parallel. */ public JobEdge(IntermediateDataSet source, JobVertex target, DistributionPattern distributionPattern) { if (source == null || target == null || distributionPattern == null) { throw new NullPointerException(); } this.target = target; this.distributionPattern = distributionPattern; this.source = source; this.sourceId = source.getId(); }
public JobEdge connectDataSetAsInput(IntermediateDataSet dataSet, DistributionPattern distPattern) { JobEdge edge = new JobEdge(dataSet, this, distPattern); this.inputs.add(edge); dataSet.addConsumer(edge); return edge; }
} else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName()));
public IntermediateDataSet createAndAddResultDataSet( IntermediateDataSetID id, ResultPartitionType partitionType) { IntermediateDataSet result = new IntermediateDataSet(id, partitionType, this); this.results.add(result); return result; }
JobVertex predecessor = edge.getSource().getProducer(); gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
result.getId(), this, numTaskVertices, result.getResultType());
private void addNodesThatHaveNoNewPredecessors(JobVertex start, List<JobVertex> target, Set<JobVertex> remaining) { // forward traverse over all produced data sets and all their consumers for (IntermediateDataSet dataSet : start.getProducedDataSets()) { for (JobEdge edge : dataSet.getConsumers()) { // a vertex can be added, if it has no predecessors that are still in the 'remaining' set JobVertex v = edge.getTarget(); if (!remaining.contains(v)) { continue; } boolean hasNewPredecessors = false; for (JobEdge e : v.getInputs()) { // skip the edge through which we came if (e == edge) { continue; } IntermediateDataSet source = e.getSource(); if (remaining.contains(source.getProducer())) { hasNewPredecessors = true; break; } } if (!hasNewPredecessors) { target.add(v); remaining.remove(v); addNodesThatHaveNoNewPredecessors(v, target, remaining); } } } }