public Query(final QueryMasterTask.QueryMasterTaskContext context, final QueryId id, final long appSubmitTime, final String queryStr, final EventHandler eventHandler, final MasterPlan plan) { this.context = context; this.systemConf = context.getConf(); this.id = id; this.clock = context.getClock(); this.appSubmitTime = appSubmitTime; this.queryStr = queryStr; this.stages = Maps.newConcurrentMap(); this.eventHandler = eventHandler; this.plan = plan; this.cursor = new ExecutionBlockCursor(plan, true); StringBuilder sb = new StringBuilder("\n======================================================="); sb.append("\nThe order of execution: \n"); int order = 1; for (ExecutionBlock currentEB : cursor) { sb.append("\n").append(order).append(": ").append(currentEB.getId()); order++; } sb.append("\n======================================================="); LOG.info(sb); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); stateMachine = stateMachineFactory.make(this); queryState = stateMachine.getCurrentState(); }
public Query(final QueryMasterTask.QueryMasterTaskContext context, final QueryId id, final long appSubmitTime, final String queryStr, final EventHandler eventHandler, final MasterPlan plan) { this.context = context; this.systemConf = context.getConf(); this.id = id; this.clock = context.getClock(); this.appSubmitTime = appSubmitTime; this.queryStr = queryStr; this.stages = Maps.newConcurrentMap(); this.eventHandler = eventHandler; this.plan = plan; this.cursor = new ExecutionBlockCursor(plan, true); StringBuilder sb = new StringBuilder("\n======================================================="); sb.append("\nThe order of execution: \n"); int order = 1; for (ExecutionBlock currentEB : cursor) { sb.append("\n").append(order).append(": ").append(currentEB.getId()); order++; } sb.append("\n======================================================="); LOG.info(sb); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); stateMachine = stateMachineFactory.make(this); queryState = stateMachine.getCurrentState(); }
public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, ExecutionBlock block) { this.context = context; this.masterPlan = masterPlan; this.block = block; this.eventHandler = context.getEventHandler(); this.rpcParams = RpcParameterFactory.get(context.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); stateMachine = stateMachineFactory.make(this); stageState = stateMachine.getCurrentState(); }
StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); String intermediateDataFormat = schedulerContext.getMasterContext().getConf().getVar(ConfVars.SHUFFLE_FILE_FORMAT); for (int i = 0; i < scans.length; i++) {
StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); String intermediateDataFormat = schedulerContext.getMasterContext().getConf().getVar(ConfVars.SHUFFLE_FILE_FORMAT); for (int i = 0; i < scans.length; i++) {
stage.getMasterPlan().getContext().getInt(SessionVars.TABLE_PARTITION_PER_SHUFFLE_SIZE); long pageSize = ((long)StorageUnit.MB) * stage.getContext().getConf().getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME); // in bytes if (pageSize >= splitVolume) { throw new RuntimeException("tajo.dist-query.table-partition.task-volume-mb should be great than " +
stage.getMasterPlan().getContext().getInt(SessionVars.TABLE_PARTITION_PER_SHUFFLE_SIZE); long pageSize = ((long)StorageUnit.MB) * stage.getContext().getConf().getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME); // in bytes if (pageSize >= splitVolume) { throw new RuntimeException("tajo.dist-query.table-partition.task-volume-mb should be great than " +
TajoConf conf = stage.context.getConf(); stage.schedulerContext.setTaskSize(conf.getIntVar(ConfVars.TASK_DEFAULT_SIZE) * 1024 * 1024); int estimatedTaskNum = (int) Math.ceil((double) table.getStats().getNumBytes() /
public static Task newEmptyTask(TaskSchedulerContext schedulerContext, TaskAttemptScheduleContext taskContext, Stage stage, int taskId) { ExecutionBlock execBlock = stage.getBlock(); Task unit = new Task(schedulerContext.getMasterContext().getConf(), taskContext, QueryIdFactory.newTaskId(schedulerContext.getBlockId(), taskId), schedulerContext.isLeafQuery(), stage.eventHandler); unit.setLogicalPlan(execBlock.getPlan()); stage.addTask(unit); return unit; }
public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, ExecutionBlock block) { this.context = context; this.masterPlan = masterPlan; this.block = block; this.eventHandler = context.getEventHandler(); this.rpcParams = RpcParameterFactory.get(context.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); stateMachine = stateMachineFactory.make(this); stageState = stateMachine.getCurrentState(); }
private void initTaskScheduler(Stage stage) throws IOException { TajoConf conf = stage.context.getConf(); stage.schedulerContext = new TaskSchedulerContext(stage.context, stage.getMasterPlan().isLeaf(stage.getId()), stage.getId()); stage.taskScheduler = TaskSchedulerFactory.get(conf, stage.schedulerContext, stage); stage.taskScheduler.init(conf); LOG.info(stage.taskScheduler.getName() + " is chosen for the task scheduling for " + stage.getId()); }
public static Task newEmptyTask(TaskSchedulerContext schedulerContext, TaskAttemptScheduleContext taskContext, Stage stage, int taskId) { ExecutionBlock execBlock = stage.getBlock(); Task unit = new Task(schedulerContext.getMasterContext().getConf(), taskContext, QueryIdFactory.newTaskId(schedulerContext.getBlockId(), taskId), schedulerContext.isLeafQuery(), stage.eventHandler); unit.setLogicalPlan(execBlock.getPlan()); stage.addTask(unit); return unit; }
private void initTaskScheduler(Stage stage) throws IOException { TajoConf conf = stage.context.getConf(); stage.schedulerContext = new TaskSchedulerContext(stage.context, stage.getMasterPlan().isLeaf(stage.getId()), stage.getId()); stage.taskScheduler = TaskSchedulerFactory.get(conf, stage.schedulerContext, stage); stage.taskScheduler.init(conf); LOG.info(stage.taskScheduler.getName() + " is chosen for the task scheduling for " + stage.getId()); }