private void initTaskScheduler(Stage stage) throws IOException { TajoConf conf = stage.context.getConf(); stage.schedulerContext = new TaskSchedulerContext(stage.context, stage.getMasterPlan().isLeaf(stage.getId()), stage.getId()); stage.taskScheduler = TaskSchedulerFactory.get(conf, stage.schedulerContext, stage); stage.taskScheduler.init(conf); LOG.info(stage.taskScheduler.getName() + " is chosen for the task scheduling for " + stage.getId()); }
public static Task newEmptyTask(TaskSchedulerContext schedulerContext, TaskAttemptScheduleContext taskContext, Stage stage, int taskId) { ExecutionBlock execBlock = stage.getBlock(); Task unit = new Task(schedulerContext.getMasterContext().getConf(), taskContext, QueryIdFactory.newTaskId(schedulerContext.getBlockId(), taskId), schedulerContext.isLeafQuery(), stage.eventHandler); unit.setLogicalPlan(execBlock.getPlan()); stage.addTask(unit); return unit; }
stage.schedulerContext.setEstimatedTaskNum(fragments.size()); } else { TajoConf conf = stage.context.getConf(); stage.schedulerContext.setTaskSize(conf.getIntVar(ConfVars.TASK_DEFAULT_SIZE) * 1024 * 1024); int estimatedTaskNum = (int) Math.ceil((double) table.getStats().getNumBytes() / (double) stage.schedulerContext.getTaskSize()); stage.schedulerContext.setEstimatedTaskNum(estimatedTaskNum);
schedulerContext.setTaskSize((int) Math.ceil((double) bothFetchSize / joinTaskNum)); schedulerContext.setEstimatedTaskNum(joinTaskNum);
schedulerContext.setEstimatedTaskNum(fetches.size());
StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); String intermediateDataFormat = schedulerContext.getMasterContext().getConf().getVar(ConfVars.SHUFFLE_FILE_FORMAT); for (int i = 0; i < scans.length; i++) {
context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker(); NettyClientBase tmClient = RpcClientManager.getInstance(). getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcParams); .setNumContainers(requestContainerNum) .setPriority(stage.getPriority()) .setQueryId(context.getMasterContext().getQueryId().getProto()) .setType(isLeaf ? ResourceType.LEAF : ResourceType.INTERMEDIATE) .setUserId(context.getMasterContext().getQueryContext().getUser()) .setRunningTasks(stage.getTotalScheduledObjectsCount() - stage.getCompletedTaskCount()) .addAllCandidateNodes(candidateWorkers) .setQueue(context.getMasterContext().getQueryContext().get("queue", "default")); //TODO set queue taskRequestEvents.add(new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId()));
if (event instanceof FragmentScheduleEvent) { FragmentScheduleEvent castEvent = (FragmentScheduleEvent) event; if (context.isLeafQuery()) { TaskAttemptScheduleContext taskContext = new TaskAttemptScheduleContext(); Task task = Stage.newEmptyTask(context, taskContext, stage, nextTaskId++); } else if (event instanceof TaskAttemptToSchedulerEvent) { TaskAttemptToSchedulerEvent castEvent = (TaskAttemptToSchedulerEvent) event; if (context.isLeafQuery()) { scheduledRequests.addLeafTask(castEvent); } else {
schedulerContext.setEstimatedTaskNum(fetches.size());
StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); String intermediateDataFormat = schedulerContext.getMasterContext().getConf().getVar(ConfVars.SHUFFLE_FILE_FORMAT); for (int i = 0; i < scans.length; i++) {
context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker(); NettyClientBase tmClient = RpcClientManager.getInstance(). getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcParams); .setNumContainers(requestContainerNum) .setPriority(stage.getPriority()) .setQueryId(context.getMasterContext().getQueryId().getProto()) .setType(isLeaf ? ResourceType.LEAF : ResourceType.INTERMEDIATE) .setUserId(context.getMasterContext().getQueryContext().getUser()) .setRunningTasks(stage.getTotalScheduledObjectsCount() - stage.getCompletedTaskCount()) .addAllCandidateNodes(candidateWorkers) .setQueue(context.getMasterContext().getQueryContext().get("queue", "default")); //TODO set queue taskRequestEvents.add(new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId()));
schedulerContext.setTaskSize((int) Math.ceil((double) bothFetchSize / joinTaskNum)); schedulerContext.setEstimatedTaskNum(joinTaskNum);
if (event instanceof FragmentScheduleEvent) { FragmentScheduleEvent castEvent = (FragmentScheduleEvent) event; if (context.isLeafQuery()) { TaskAttemptScheduleContext taskContext = new TaskAttemptScheduleContext(); Task task = Stage.newEmptyTask(context, taskContext, stage, nextTaskId++); } else if (event instanceof TaskAttemptToSchedulerEvent) { TaskAttemptToSchedulerEvent castEvent = (TaskAttemptToSchedulerEvent) event; if (context.isLeafQuery()) { scheduledRequests.addLeafTask(castEvent); } else {
public static Task newEmptyTask(TaskSchedulerContext schedulerContext, TaskAttemptScheduleContext taskContext, Stage stage, int taskId) { ExecutionBlock execBlock = stage.getBlock(); Task unit = new Task(schedulerContext.getMasterContext().getConf(), taskContext, QueryIdFactory.newTaskId(schedulerContext.getBlockId(), taskId), schedulerContext.isLeafQuery(), stage.eventHandler); unit.setLogicalPlan(execBlock.getPlan()); stage.addTask(unit); return unit; }
private static void scheduleFragmentsForLeafQuery(Stage stage) throws IOException, TajoException { ExecutionBlock execBlock = stage.getBlock(); ScanNode[] scans = execBlock.getScanNodes(); Preconditions.checkArgument(scans.length == 1, "Must be Scan Query"); ScanNode scan = scans[0]; TableDesc table = stage.context.getTableDesc(scan); Collection<Fragment> fragments = SplitUtil.getSplits( TablespaceManager.get(scan.getTableDesc().getUri()), scan, table, false); SplitUtil.preparePartitionScanPlanForSchedule(scan); Stage.scheduleFragments(stage, fragments); // The number of leaf tasks should be the number of fragments. stage.schedulerContext.setEstimatedTaskNum(fragments.size()); } }
protected void cancel(TaskAttempt taskAttempt) { TaskAttemptToSchedulerEvent schedulerEvent = new TaskAttemptToSchedulerEvent( EventType.T_SCHEDULE, taskAttempt.getTask().getId().getExecutionBlockId(), null, taskAttempt); if(taskAttempt.isLeafTask()) { releaseTaskAttempt(taskAttempt); scheduledRequests.addLeafTask(schedulerEvent); } else { scheduledRequests.addNonLeafTask(schedulerEvent); } context.getMasterContext().getEventHandler().handle( new TaskAttemptEvent(taskAttempt.getId(), TaskAttemptEventType.TA_ASSIGN_CANCEL)); }
private void initTaskScheduler(Stage stage) throws IOException { TajoConf conf = stage.context.getConf(); stage.schedulerContext = new TaskSchedulerContext(stage.context, stage.getMasterPlan().isLeaf(stage.getId()), stage.getId()); stage.taskScheduler = TaskSchedulerFactory.get(conf, stage.schedulerContext, stage); stage.taskScheduler.init(conf); LOG.info(stage.taskScheduler.getName() + " is chosen for the task scheduling for " + stage.getId()); }
schedulerContext.setEstimatedTaskNum(baseFragments.size());
public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) throws InterruptedException { Collections.shuffle(taskRequests); LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<TaskRequestEvent>(); String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext(). getConnectionInfo().getHostAndQMPort(); WorkerConnectionInfo connectionInfo = context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); if(connectionInfo == null) continue; int nodes = context.getMasterContext().getWorkerMap().size(); false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), queryMasterHostAndPort); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
scan.getTableName()); } else { schedulerContext.setEstimatedTaskNum(determinedTaskNum);