/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
@Override public ResultPartitionStatus getResultPartitionStatus(IntermediateDataSetID resultID, int partitionNumber) { checkNotNull(resultID); IntermediateResult result = executionGraph.getAllIntermediateResults().get(resultID); if (result == null) { throw new IllegalArgumentException("Cannot find any result with id " + resultID); } return new ResultPartitionStatus( resultID, partitionNumber, result.getPartitions()[partitionNumber].isConsumable()); }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
ResultPartitionType getResultType() { return totalResult.getResultType(); }
final IntermediateDataSet result = jobVertex.getProducedDataSets().get(i); this.producedDataSets[i] = new IntermediateResult( result.getId(), this, if (ir.getNumberOfAssignedPartitions() != parallelism) { throw new RuntimeException("The intermediate result's partitions were not correctly assigned.");
int consumerIndex = ires.registerConsumer(); executionEdges.add(new ArrayList<>()); for (int i = 0; i < ires.getPartitions().length; i++) { IntermediateResultPartition partition = ires.getPartitions()[i]; Collection<ExecutionVertexID> consumerExecutionVertices = edge.getConsumerExecutionVertices(i);
public IntermediateDataSetID getResultId() { return intermediateResult.getId(); }
@Override public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { // use the yarn shuffle service data port Integer dataPort = configuration.getInteger( ExternalBlockShuffleServiceOptions.FLINK_SHUFFLE_SERVICE_PORT_KEY); // use the taskmanager ip address, for the shuffle service deployed on the same host // of the taskmanager is used to shuffle data to down streams. InetSocketAddress address = new InetSocketAddress(producerLocation.address(), dataPort); ConnectionID connectionId = new ConnectionID(address, intermediateResult.getConnectionIndex()); return ResultPartitionLocation.createRemote(connectionId); } }
@Override public CompletableFuture<ExecutionState> requestPartitionState( final IntermediateDataSetID intermediateResultId, final ResultPartitionID resultPartitionId) { final Execution execution = executionGraph.getRegisteredExecutions().get(resultPartitionId.getProducerId()); if (execution != null) { return CompletableFuture.completedFuture(execution.getState()); } else { final IntermediateResult intermediateResult = executionGraph.getAllIntermediateResults().get(intermediateResultId); if (intermediateResult != null) { // Try to find the producing execution Execution producerExecution = intermediateResult .getPartitionById(resultPartitionId.getPartitionId()) .getProducer() .getCurrentExecutionAttempt(); if (producerExecution.getAttemptId().equals(resultPartitionId.getProducerId())) { return CompletableFuture.completedFuture(producerExecution.getState()); } else { return FutureUtils.completedExceptionally(new PartitionProducerDisposedException(resultPartitionId)); } } else { return FutureUtils.completedExceptionally(new IllegalArgumentException("Intermediate data set with ID " + intermediateResultId + " not found.")); } } }
result.setPartition(subTaskIndex, irp);
int consumerIndex = ires.registerConsumer();
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
public ResultPartitionType getResultType() { return totalResult.getResultType(); }
final IntermediateDataSet result = jobVertex.getProducedDataSets().get(i); this.producedDataSets[i] = new IntermediateResult( result.getId(), this, if (ir.getNumberOfAssignedPartitions() != parallelism) { throw new RuntimeException("The intermediate result's partitions were not correctly assigned.");
public void attachJobGraph(List<JobVertex> topologicallySorted) throws JobException { LOG.debug("Attaching {} topologically sorted vertices to existing job graph with {} " + "vertices and {} intermediate results.", topologicallySorted.size(), tasks.size(), intermediateResults.size()); final ArrayList<ExecutionJobVertex> newExecJobVertices = new ArrayList<>(topologicallySorted.size()); createExecutionJobVertex(topologicallySorted); for (JobVertex jobVertex : topologicallySorted) { if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; } ExecutionJobVertex ejv = tasks.get(jobVertex.getID()); ejv.connectToPredecessors(this.intermediateResults); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); } } this.verticesInCreationOrder.add(ejv); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); } terminationFuture = new CompletableFuture<>(); failoverStrategy.notifyNewVertices(newExecJobVertices); }
@Override public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { if (consumerLocation.getResourceID().equals(producerLocation.getResourceID())) { // in the same taskmanager return ResultPartitionLocation.createLocal(); } else { // use the taskmanager ip and data port ConnectionID connectionId = new ConnectionID(producerLocation, intermediateResult.getConnectionIndex()); return ResultPartitionLocation.createRemote(connectionId); } } }
@Override public CompletableFuture<ExecutionState> requestPartitionState( final IntermediateDataSetID intermediateResultId, final ResultPartitionID resultPartitionId) { final Execution execution = executionGraph.getRegisteredExecutions().get(resultPartitionId.getProducerId()); if (execution != null) { return CompletableFuture.completedFuture(execution.getState()); } else { final IntermediateResult intermediateResult = executionGraph.getAllIntermediateResults().get(intermediateResultId); if (intermediateResult != null) { // Try to find the producing execution Execution producerExecution = intermediateResult .getPartitionById(resultPartitionId.getPartitionId()) .getProducer() .getCurrentExecutionAttempt(); if (producerExecution.getAttemptId().equals(resultPartitionId.getProducerId())) { return CompletableFuture.completedFuture(producerExecution.getState()); } else { return FutureUtils.completedExceptionally(new PartitionProducerDisposedException(resultPartitionId)); } } else { return FutureUtils.completedExceptionally(new IllegalArgumentException("Intermediate data set with ID " + intermediateResultId + " not found.")); } } }
result.setPartition(subTaskIndex, irp);
int consumerIndex = ires.registerConsumer();
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }