public TaskAttempt newAttempt() { TaskAttempt attempt = new TaskAttempt(scheduleContext, QueryIdFactory.newTaskAttemptId(this.getId(), ++nextAttempt), this, eventHandler); lastAttemptId = attempt.getId(); return attempt; }
private TableStats[] computeStatFromTasks() { List<TableStats> inputStatsList = Lists.newArrayList(); List<TableStats> resultStatsList = Lists.newArrayList(); for (Task unit : getTasks()) { resultStatsList.add(unit.getStats()); if (unit.getLastAttempt().getInputStats() != null) { inputStatsList.add(unit.getLastAttempt().getInputStats()); } } TableStats inputStats = StatisticsUtil.aggregateTableStat(inputStatsList); TableStats resultStats = StatisticsUtil.aggregateTableStat(resultStatsList); return new TableStats[]{inputStats, resultStats}; }
public Set<PartitionDescProto> getPartitions() { Set<PartitionDescProto> partitions = new HashSet<>(); for(Task eachTask : getTasks()) { if (eachTask.getLastAttempt() != null && !eachTask.getLastAttempt().getPartitions().isEmpty()) { partitions.addAll(eachTask.getLastAttempt().getPartitions()); } } return partitions; }
public void releaseTaskAttempt(TaskAttempt taskAttempt) { if (taskAttempt != null && taskAttempt.isLeafTask() && taskAttempt.getWorkerConnectionInfo() != null) { HostVolumeMapping mapping = scheduledRequests.leafTaskHostMapping.get(taskAttempt.getWorkerConnectionInfo().getHost()); if (mapping != null && mapping.lastAssignedVolumeId.containsKey(taskAttempt.getId())) { mapping.decreaseConcurrency(mapping.lastAssignedVolumeId.remove(taskAttempt.getId())); } } } /**
protected void cancel(TaskAttempt taskAttempt) { TaskAttemptToSchedulerEvent schedulerEvent = new TaskAttemptToSchedulerEvent( EventType.T_SCHEDULE, taskAttempt.getTask().getId().getExecutionBlockId(), null, taskAttempt); if(taskAttempt.isLeafTask()) { releaseTaskAttempt(taskAttempt); scheduledRequests.addLeafTask(schedulerEvent); } else { scheduledRequests.addNonLeafTask(schedulerEvent); } context.getMasterContext().getEventHandler().handle( new TaskAttemptEvent(taskAttempt.getId(), TaskAttemptEventType.TA_ASSIGN_CANCEL)); }
private void fillTaskStatistics(TaskCompletionReport report) { this.progress = 1.0f; List<IntermediateEntry> partitions = new ArrayList<IntermediateEntry>(); if (report.getShuffleFileOutputsCount() > 0) { this.getTask().setShuffleFileOutputs(report.getShuffleFileOutputsList()); PullHost host = new PullHost(getWorkerConnectionInfo().getHost(), getWorkerConnectionInfo().getPullServerPort()); for (ShuffleFileOutput p : report.getShuffleFileOutputsList()) { IntermediateEntry entry = new IntermediateEntry(getId().getTaskId().getId(), getId().getId(), p.getPartId(), host, p.getVolume()); partitions.add(entry); } } this.getTask().setIntermediateData(partitions); if (report.hasInputStats()) { this.inputStats = report.getInputStats(); } if (report.hasResultStats()) { this.resultStats = report.getResultStats(); this.getTask().setStats(new TableStats(resultStats)); } }
taskHistory.setId(lastAttempt.getId().toString()); taskHistory.setState(lastAttempt.getState().toString()); taskHistory.setProgress(lastAttempt.getProgress());
@Override public void transition(TaskAttempt taskAttempt, TaskAttemptEvent event) { taskAttempt.eventHandler.handle(new LocalTaskEvent(taskAttempt.getId(), taskAttempt.getWorkerConnectionInfo().getId(), LocalTaskEventType.KILL)); } }
@Override public void transition(TaskAttempt taskAttempt, TaskAttemptEvent event) { if (!(event instanceof TaskCompletionEvent)) { throw new IllegalArgumentException("event should be a TaskCompletionEvent type."); } TaskCompletionReport report = ((TaskCompletionEvent)event).getReport(); try { if (report.getPartitionsCount() > 0) { taskAttempt.addPartitions(report.getPartitionsList()); } taskAttempt.fillTaskStatistics(report); taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_SUCCEEDED)); } catch (Throwable t) { taskAttempt.eventHandler.handle(new TaskFatalErrorEvent(taskAttempt.getId(), t)); taskAttempt.addDiagnosticInfo(ExceptionUtils.getStackTrace(t)); } } }
@Override public void statusUpdate(RpcController controller, TaskStatusProto request, RpcCallback<PrimitiveProtos.NullProto> done) { QueryId queryId = new QueryId(request.getId().getTaskId().getExecutionBlockId().getQueryId()); TaskAttemptId attemptId = new TaskAttemptId(request.getId()); QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(queryId); if (queryMasterTask != null) { Stage sq = queryMasterTask.getQuery().getStage(attemptId.getTaskId().getExecutionBlockId()); Task task = sq.getTask(attemptId.getTaskId()); TaskAttempt attempt = task.getAttempt(attemptId.getId()); if(LOG.isDebugEnabled()){ LOG.debug(String.format("Task State: %s, Attempt State: %s", task.getState().name(), attempt.getState().name())); } if (request.getState() == TajoProtos.TaskAttemptState.TA_KILLED) { LOG.warn(attemptId + " Killed"); attempt.handle( new TaskAttemptEvent(new TaskAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED)); } else { queryMasterTask.getEventHandler().handle( new TaskAttemptStatusUpdateEvent(new TaskAttemptId(request.getId()), request)); } } done.run(TajoWorker.NULL_PROTO); }
TableStats inputStats = eachTask.getLastAttempt().getInputStats(); if (inputStats != null) { totalInputBytes += inputStats.getNumBytes(); totalReadRows += inputStats.getNumRows(); TableStats outputStats = eachTask.getLastAttempt().getResultStats(); if (outputStats != null) { totalWriteBytes += outputStats.getNumBytes();
scheduledRequests.leafTasks.remove(castedEvent.getTaskAttempt().getId()); LOG.info(castedEvent.getTaskAttempt().getId() + " is canceled from " + this.getClass().getSimpleName()); ((TaskAttemptToSchedulerEvent) event).getTaskAttempt().handle( new TaskAttemptEvent(castedEvent.getTaskAttempt().getId(), TaskAttemptEventType.TA_SCHEDULE_CANCELED));
public TaskAttemptState getLastAttemptStatus() { TaskAttempt lastAttempt = getLastAttempt(); if (lastAttempt != null) { return lastAttempt.getState(); } else { return TaskAttemptState.TA_ASSIGNED; } }
@Override public void transition(TaskAttempt taskAttempt, TaskAttemptEvent taskAttemptEvent) { taskAttempt.eventHandler.handle(new TaskAttemptToSchedulerEvent( EventType.T_SCHEDULE_CANCEL, taskAttempt.getTask().getId().getExecutionBlockId(), taskAttempt.scheduleContext, taskAttempt)); } }
private void addNonLeafTask(TaskAttemptToSchedulerEvent event) { nonLeafTasks.add(event.getTaskAttempt().getId()); }
@Override public void transition(Task task, TaskEvent event) { if (!(event instanceof TaskTAttemptEvent)) { throw new IllegalArgumentException("event should be a TaskTAttemptEvent type."); } TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event; TaskAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId()); task.successfulAttempt = attemptEvent.getTaskAttemptId(); task.succeededWorker = attempt.getWorkerConnectionInfo(); task.finishTask(); task.eventHandler.handle(new StageTaskEvent(event.getTaskId(), TaskState.SUCCEEDED)); } }
public float getProgress() { List<Task> tempTasks = null; readLock.lock(); try { if (getState() == StageState.NEW) { return 0.0f; } else { tempTasks = new ArrayList<Task>(tasks.values()); } } finally { readLock.unlock(); } float totalProgress = 0.0f; for (Task eachTask : tempTasks) { if (eachTask.getLastAttempt() != null) { totalProgress += eachTask.getLastAttempt().getProgress(); } } if (totalProgress > 0.0f) { return (float) Math.floor((totalProgress / (float) Math.max(tempTasks.size(), 1)) * 1000.0f) / 1000.0f; } else { return 0.0f; } }
public void releaseTaskAttempt(TaskAttempt taskAttempt) { if (taskAttempt != null && taskAttempt.isLeafTask() && taskAttempt.getWorkerConnectionInfo() != null) { HostVolumeMapping mapping = scheduledRequests.leafTaskHostMapping.get(taskAttempt.getWorkerConnectionInfo().getHost()); if (mapping != null && mapping.lastAssignedVolumeId.containsKey(taskAttempt.getId())) { mapping.decreaseConcurrency(mapping.lastAssignedVolumeId.remove(taskAttempt.getId())); } } } /**