boolean isConsumable() { if (resultType.isPipelined()) { return true; } else { return numberOfRunningProducers.get() == 0; } }
boolean isConsumable() { if (resultType.isPipelined()) { return true; } else { return numberOfRunningProducers.get() == 0; } }
boolean isConsumable() { if (resultType.isPipelined()) { return true; } else { return numberOfRunningProducers.get() == 0; } }
/** * Notifies pipelined consumers of this result partition once. */ private void notifyPipelinedConsumers() { if (sendScheduleOrUpdateConsumersMessage && !hasNotifiedPipelinedConsumers && partitionType.isPipelined()) { partitionConsumableNotifier.notifyPartitionConsumable(jobId, partitionId, taskActions); hasNotifiedPipelinedConsumers = true; } } }
/** * Notifies pipelined consumers of this result partition once. */ private void notifyPipelinedConsumers() { if (sendScheduleOrUpdateConsumersMessage && !hasNotifiedPipelinedConsumers && partitionType.isPipelined()) { partitionConsumableNotifier.notifyPartitionConsumable(jobId, partitionId, taskActions); hasNotifiedPipelinedConsumers = true; } } }
/** * Notifies pipelined consumers of this result partition once. */ private void notifyPipelinedConsumers() { if (sendScheduleOrUpdateConsumersMessage && !hasNotifiedPipelinedConsumers && partitionType.isPipelined()) { partitionConsumableNotifier.notifyPartitionConsumable(jobId, partitionId, taskActions); hasNotifiedPipelinedConsumers = true; } } }
public boolean isConsumable() { if (getResultType().isPipelined()) { return dataProduced; } else { return isFinished; } }
/** * Notifies pipelined consumers of this result partition once. */ private void notifyPipelinedConsumers() { if (sendScheduleOrUpdateConsumersMessage && !hasNotifiedPipelinedConsumers && partitionType.isPipelined()) { partitionConsumableNotifier.notifyPartitionConsumable(jobId, partitionId, taskActions); hasNotifiedPipelinedConsumers = true; } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition partition.markDataProduced(); // Notify the scheduler to handle the consumable partition notifyAndUpdateConsumers(partition); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
public void notifyResultPartitionConsumable( ExecutionVertexID executionVertexID, IntermediateDataSetID resultID, int partitionNumber, TaskManagerLocation location) { if (!operationLogManager.isReplaying() && executionGraph.getAllIntermediateResults().get(resultID).getResultType().isPipelined()) { operationLogManager.writeOpLog(new ResultPartitionOperationLog( executionVertexID, resultID, location)); } graphManagerPlugin.onResultPartitionConsumable( new ResultPartitionConsumableEvent(resultID, partitionNumber)); }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
if (input.getResultType().isPipelined()) { hasPipelinedInputs = true; break; if (inputs.get(inputNum).getResultType().isPipelined()) {
if (input.getResultType().isPipelined()) { hasPipelinedInputs = true; break; if (inputs.get(inputNum).getResultType().isPipelined()) {
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
if (input.getResultType().isPipelined()) { hasPipelinedInputs = true; break; if (inputs.get(inputNum).getResultType().isPipelined()) {
if (partition.getResultType().isPipelined()) { if (partition.hasDataProduced() != partitionsConsumable[i]) { LOG.info("Reconcile execution {} failed due to partition {} consumable not equals to {}.",
vertexID.getJobVertexID(), resultID, jobGraph.getResult(resultID).getResultType().isPipelined());
/** * Recover the pipelined result partition consume status after job master failover. * * @param resultId The intermediate data set id in the log. */ public void recoverResultPartitionStatus( IntermediateDataSetID resultId, TaskManagerLocation location) { IntermediateResultPartition partitionToRecover = null; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (irp.getIntermediateResult().getId().equals(resultId)) { partitionToRecover = irp; } } if (partitionToRecover == null) { throw new FlinkRuntimeException("Can not find the intermediate result " + resultId + " on " + getTaskNameWithSubtaskIndex()); } if (!(ExecutionState.RUNNING.equals(currentExecution.getState()) && partitionToRecover.getResultType().isPipelined())) { throw new FlinkRuntimeException("Invalid state " + currentExecution.getState() + " for " + getTaskNameWithSubtaskIndex()); } currentExecution.getTaskManagerLocationFuture().complete(location); scheduleOrUpdateConsumers(new ResultPartitionID(partitionToRecover.getPartitionId(), currentExecution.getAttemptId())); }
if ((consumedPartition.getResultType().isPipelined() || consumedPartition.isConsumable()) && producerSlot != null &&