/** * Increase the count of running tasks and disk loads for a certain task runner. * * @param volumeId Volume identifier * @return the volume load (i.e., how many running tasks use this volume) */ private synchronized int increaseConcurrency(int volumeId) { int concurrency = 1; if (diskVolumeLoads.containsKey(volumeId)) { concurrency = diskVolumeLoads.get(volumeId) + 1; } if (volumeId > -1) { info(LOG, "Assigned host : " + host + ", Volume : " + volumeId + ", Concurrency : " + concurrency); } else if (volumeId == -1) { // this case is disabled namenode block meta or compressed text file or amazon s3 info(LOG, "Assigned host : " + host + ", Unknown Volume : " + volumeId + ", Concurrency : " + concurrency); } else if (volumeId == REMOTE) { // this case has processed all block on host and it will be assigned to remote info(LOG, "Assigned host : " + host + ", Remaining local tasks : " + getRemainingLocalTaskSize() + ", Remote Concurrency : " + concurrency + ", Unassigned volumes: " + unassignedTaskForEachVolume.size()); } diskVolumeLoads.put(volumeId, concurrency); return concurrency; }
/** * Increase the count of running tasks and disk loads for a certain task runner. * * @param volumeId Volume identifier * @return the volume load (i.e., how many running tasks use this volume) */ private synchronized int increaseConcurrency(int volumeId) { int concurrency = 1; if (diskVolumeLoads.containsKey(volumeId)) { concurrency = diskVolumeLoads.get(volumeId) + 1; } if (volumeId > DataLocation.UNKNOWN_VOLUME_ID) { info(LOG, "Assigned host : " + host + ", Volume : " + volumeId + ", Concurrency : " + concurrency); } else if (volumeId == DataLocation.UNKNOWN_VOLUME_ID) { // this case is disabled namenode block meta or compressed text file or amazon s3 info(LOG, "Assigned host : " + host + ", Unknown Volume : " + volumeId + ", Concurrency : " + concurrency); } else if (volumeId == DataLocation.REMOTE_VOLUME_ID) { // this case has processed all block on host and it will be assigned to remote info(LOG, "Assigned host : " + host + ", Remaining local tasks : " + getRemainingLocalTaskSize() + ", Remote Concurrency : " + concurrency + ", Unassigned volumes: " + unassignedTaskForEachVolume.size()); } diskVolumeLoads.put(volumeId, concurrency); return concurrency; }
@Override public void stop() { isStopped = true; if (schedulingThread != null) { synchronized (schedulingThread) { schedulingThread.interrupt(); } } candidateWorkers.clear(); scheduledRequests.clear(); info(LOG, "Task Scheduler stopped"); super.stop(); }
@Override public void stop() { isStopped = true; if (schedulingThread != null) { synchronized (schedulingThread) { schedulingThread.interrupt(); } } candidateWorkers.clear(); scheduledRequests.clear(); info(LOG, "Task Scheduler stopped"); super.stop(); }
public void run() { while (!isStopped && !Thread.currentThread().isInterrupted()) { try { schedule(); } catch (InterruptedException e) { if (isStopped) { break; } else { LOG.fatal(e.getMessage(), e); stage.abort(StageState.ERROR, e); } } catch (Throwable e) { LOG.fatal(e.getMessage(), e); stage.abort(StageState.ERROR, e); break; } } info(LOG, "TaskScheduler schedulingThread stopped"); } };
public void run() { while (!isStopped && !Thread.currentThread().isInterrupted()) { try { schedule(); } catch (InterruptedException e) { if (isStopped) { break; } else { LOG.fatal(e.getMessage(), e); stage.abort(StageState.ERROR, e); } } catch (Throwable e) { LOG.fatal(e.getMessage(), e); stage.abort(StageState.ERROR, e); break; } } info(LOG, "TaskScheduler schedulingThread stopped"); } };
@Override public void start() { info(LOG, "Start TaskScheduler"); maximumRequestContainer = Math.min(tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_REQUEST_MAX_NUM) , stage.getContext().getWorkerMap().size()); if (isLeaf) { candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts())); } else { //find assigned hosts for Non-Leaf locality in children executionBlock List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock()); for (ExecutionBlock executionBlock : executionBlockList) { Stage childStage = stage.getContext().getStage(executionBlock.getId()); candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet()); } } this.schedulingThread.start(); super.start(); }
@Override public void start() { info(LOG, "Start TaskScheduler"); maximumRequestContainer = Math.min(tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_REQUEST_MAX_NUM) , stage.getContext().getWorkerMap().size()); if (isLeaf) { candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts())); } else { //find assigned hosts for Non-Leaf locality in children executionBlock List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock()); for (ExecutionBlock executionBlock : executionBlockList) { Stage childStage = stage.getContext().getStage(executionBlock.getId()); candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet()); } } this.schedulingThread.start(); super.start(); }
info(LOG, "Canceled requests: " + responseProto.getCancellationTaskCount() + " from " + addr); continue;
info(LOG, "Canceled requests: " + responseProto.getCancellationTaskCount() + " from " + addr); continue;