@Override public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { // use the yarn shuffle service data port Integer dataPort = configuration.getInteger( ExternalBlockShuffleServiceOptions.FLINK_SHUFFLE_SERVICE_PORT_KEY); // use the taskmanager ip address, for the shuffle service deployed on the same host // of the taskmanager is used to shuffle data to down streams. InetSocketAddress address = new InetSocketAddress(producerLocation.address(), dataPort); ConnectionID connectionId = new ConnectionID(address, intermediateResult.getConnectionIndex()); return ResultPartitionLocation.createRemote(connectionId); } }
@Override public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { if (consumerLocation.getResourceID().equals(producerLocation.getResourceID())) { // in the same taskmanager return ResultPartitionLocation.createLocal(); } else { // use the taskmanager ip and data port ConnectionID connectionId = new ConnectionID(producerLocation, intermediateResult.getConnectionIndex()); return ResultPartitionLocation.createRemote(connectionId); } } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
partition.getIntermediateResult().getConnectionIndex());
partition.getIntermediateResult().getConnectionIndex());
consumedPartition.getIntermediateResult().getConnectionIndex());
consumedPartition.getIntermediateResult().getConnectionIndex());
partition.getIntermediateResult().getConnectionIndex());
consumedPartition.getIntermediateResult().getConnectionIndex());