jobVertex.setParallelism(parallelism); } else { parallelism = jobVertex.getParallelism();
numChannelsDynamicPath++; numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern, sourceVertex.getParallelism(), targetVertex.getParallelism());
assertEquals(1, verticesSorted.get(0).getParallelism()); assertEquals(1, verticesSorted.get(1).getParallelism());
JobVertex sinkVertex = jobVertices.get(2); assertEquals(2, sourceVertex.getParallelism()); assertEquals(4, mapVertex.getParallelism()); assertEquals(2, sinkVertex.getParallelism());
syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
public int getParallelism() { return jobVertex.getParallelism(); }
public boolean allTasksDeploying() { return deployedNum.get() >= jobVertex.getParallelism(); }
/** * Gets the maximum parallelism of all operations in this job graph. * * @return The maximum parallelism of this job graph */ public int getMaximumParallelism() { int maxParallelism = -1; for (JobVertex vertex : taskVertices.values()) { maxParallelism = Math.max(vertex.getParallelism(), maxParallelism); } return maxParallelism; }
/** * Gets the maximum parallelism of all operations in this job graph. * * @return The maximum parallelism of this job graph */ public int getMaximumParallelism() { int maxParallelism = -1; for (JobVertex vertex : taskVertices.values()) { maxParallelism = Math.max(vertex.getParallelism(), maxParallelism); } return maxParallelism; }
/** * Gets the maximum parallelism of all operations in this job graph. * * @return The maximum parallelism of this job graph */ public int getMaximumParallelism() { int maxParallelism = -1; for (JobVertex vertex : taskVertices.values()) { maxParallelism = Math.max(vertex.getParallelism(), maxParallelism); } return maxParallelism; }
/** * Gets the maximum parallelism of all operations in this job graph. * * @return The maximum parallelism of this job graph */ public int getMaximumParallelism() { int maxParallelism = -1; for (JobVertex vertex : taskVertices.values()) { maxParallelism = Math.max(vertex.getParallelism(), maxParallelism); } return maxParallelism; }
private Collection<ExecutionVertexID> getConsumerExecutionVerticesAllToAll() { Collection<ExecutionVertexID> consumerVertices = new ArrayList<>(); for (int i = 0; i < target.getParallelism(); i++) { consumerVertices.add(new ExecutionVertexID(target.getID(), i)); } return consumerVertices; }
@Override public void onSchedulingStarted() { final List<ExecutionVertexID> verticesToSchedule = new ArrayList<>(); for (JobVertex vertex : jobGraph.getVerticesSortedTopologicallyFromSources()) { for (int i = 0; i < vertex.getParallelism(); i++) { verticesToSchedule.add(new ExecutionVertexID(vertex.getID(), i)); } } scheduler.scheduleExecutionVertices(verticesToSchedule); }
public Collection<Collection<ExecutionVertexID>> getResultPartitionConsumerExecutionVertices( IntermediateDataSetID resultID, int partitionNumber) { IntermediateDataSet result = getResult(resultID); if (result == null) { throw new IllegalArgumentException("Cannot find the given result " + resultID + " in job graph"); } if (partitionNumber >= result.getProducer().getParallelism()) { throw new IllegalArgumentException("Result partition index out of bounds: " + partitionNumber + "/" + result.getProducer().getParallelism()); } Collection<Collection<ExecutionVertexID>> consumerVertices = new ArrayList<>(); for (JobEdge edge : getResult(resultID).getConsumers()) { consumerVertices.add(edge.getConsumerExecutionVertices(partitionNumber)); } return consumerVertices; }
@Override public void onSchedulingStarted() { final List<ExecutionVertexID> verticesToSchedule = new ArrayList<>(); for (JobVertex vertex : jobGraph.getVerticesSortedTopologicallyFromSources()) { if (vertex.isInputVertex()) { for (int i = 0; i < vertex.getParallelism(); i++) { verticesToSchedule.add(new ExecutionVertexID(vertex.getID(), i)); } } } scheduleOneByOne(verticesToSchedule); }
public static void initializeOutputFormatsOnMaster( JobVertex jobVertex, AbstractFormatStub<OperatorID, ?> stub, final Map<OperatorID, String> formatDescriptions) throws RuntimeException { // set user classloader before calling user code final ClassLoader original = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(stub.getClassLoader()); try { Iterator<? extends Pair<OperatorID, ?>> it = stub.getFormat(FormatType.OUTPUT); it.forEachRemaining( (pair) -> { OperatorID key = pair.getKey(); OutputFormat outputFormat = (OutputFormat) pair.getValue(); if (outputFormat instanceof InitializeOnMaster) { try { outputFormat.configure(stub.getParameters(key)); ((InitializeOnMaster) outputFormat).initializeGlobal(jobVertex.getParallelism()); } catch (Throwable t) { throw new RuntimeException("Configuring the OutputFormat (" + "description: " + formatDescriptions.get(key) + ", stubKey: " + key + ") failed: " + t.getMessage(), t); } } } ); } finally { // restore previous classloader Thread.currentThread().setContextClassLoader(original); } }
public static void finalizeOutputFormatsOnMaster( JobVertex jobVertex, AbstractFormatStub<OperatorID, ?> stub, final Map<OperatorID, String> formatDescriptions) throws RuntimeException { // set user classloader before calling user code final ClassLoader original = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(stub.getClassLoader()); try { Iterator<? extends Pair<OperatorID, ?>> it = stub.getFormat(FormatType.OUTPUT); it.forEachRemaining( (pair) -> { OperatorID key = pair.getKey(); OutputFormat outputFormat = (OutputFormat) pair.getValue(); if (outputFormat instanceof FinalizeOnMaster) { try { outputFormat.configure(stub.getParameters(key)); ((FinalizeOnMaster) outputFormat).finalizeGlobal(jobVertex.getParallelism()); } catch (Throwable t) { throw new RuntimeException("Configuring the OutputFormat (" + "description: " + formatDescriptions.get(key) + ", stubKey: " + key + ") failed: " + t.getMessage(), t); } } } ); } finally { // restore previous classloader Thread.currentThread().setContextClassLoader(original); } }
private Collection<ExecutionVertexID> getConsumerExecutionVerticesPointwise(int partitionNumber) { final int sourceCount = source.getProducer().getParallelism(); final int targetCount = target.getParallelism();
for (int i = 0; i < producerJobVertex.getParallelism(); i++) { for (JobEdge edge : jobGraph.getResult(resultID).getConsumers()) { for (ExecutionVertexID vertexID : edge.getConsumerExecutionVertices(i)) {