private JSONObject outputPlan(Task<?> task, PrintStream out, JSONObject parentJSON, boolean extended, boolean jsonOutput, int indent) throws Exception { if (out != null) { out.print(indentString(indent)); out.print("Stage: "); out.print(task.getId()); out.print("\n"); } // Start by getting the work part of the task and call the output plan for // the work JSONObject jsonOutputPlan = outputPlan(task.getWork(), out, extended, jsonOutput, jsonOutput ? 0 : indent + 2); if (out != null) { out.println(); } if (jsonOutput) { parentJSON.put(task.getId(), jsonOutputPlan); } return null; }
/** * Makes dependentTask dependent of task. */ private void interleaveTask(Task<? extends Serializable> dependentTask, Task<? extends Serializable> task) { for (Task<? extends Serializable> parentOfStatsTask : dependentTask.getParentTasks()) { parentOfStatsTask.addDependentTask(task); } for (Task<? extends Serializable> parentOfCrtTblTask : task.getParentTasks()) { parentOfCrtTblTask.removeDependentTask(dependentTask); } task.addDependentTask(dependentTask); }
/** * Launches a task, and sets its exit value in the result variable. */ public void runSequential() { int exitVal = -101; try { exitVal = tsk.executeTask(ss == null ? null : ss.getHiveHistory()); } catch (Throwable t) { if (tsk.getException() == null) { tsk.setException(t); } LOG.error("Error in executeTask", t); } result.setExitVal(exitVal); if (tsk.getException() != null) { result.setTaskError(tsk.getException()); } }
public TaskDisplay(Task task) { taskId = task.getId(); externalHandle = task.getExternalHandle(); taskType = task.getType(); name = task.getName(); requireLock = task.requireLock(); } @JsonIgnore
MapredWork mapredWork = (MapredWork) currTask.getWork(); MapredLocalWork localwork = mapredWork.getMapWork().getMapRedLocalWork(); if (localwork != null) { Path tmpPath = Utilities.generateTmpPath(ctx.getLocalTmpPath(), currTask.getId()); localwork.setTmpPath(tmpPath); mapredWork.getMapWork().setTmpHDFSPath(Utilities.generateTmpPath( ctx.getMRTmpPath(), currTask.getId())); localTask.setBackupTask(currTask.getBackupTask()); localTask.setBackupChildrenTasks(currTask.getBackupChildrenTasks()); currTask.setBackupChildrenTasks(null); currTask.setBackupTask(null); if (currTask.getTaskTag() == Task.CONVERTED_MAPJOIN) { localTask.setTaskTag(Task.CONVERTED_MAPJOIN_LOCAL); } else { localTask.setTaskTag(Task.HINTED_MAPJOIN_LOCAL); currTask.setTaskTag(Task.HINTED_MAPJOIN); currTask.setBackupTask(localTask.getBackupTask()); currTask.setBackupChildrenTasks(localTask.getBackupChildrenTasks()); return; List<Task<? extends Serializable>> parentTasks = currTask.getParentTasks(); currTask.setParentTasks(null); if (parentTasks != null) { for (Task<? extends Serializable> tsk : parentTasks) {
public JSONObject outputDependencies(PrintStream out, boolean jsonOutput, boolean appendTaskType, List<Task> tasks) throws Exception { if (out != null) { out.println(STAGE_DEPENDENCIES + ":"); } JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null; for (Task task : tasks) { JSONObject jsonOut = outputDependencies(task, out, json, jsonOutput, appendTaskType, 2); if (jsonOutput && jsonOut != null) { json.put(task.getId(), jsonOut); } } return jsonOutput ? json : null; }
LinkedHashMap<String, String> newPartSpec = null; if (partSpec != null) { newPartSpec = new LinkedHashMap<>(partSpec); boolean isBlocking = !HiveConf.getBoolVar(conf, ConfVars.TRANSACTIONAL_CONCATENATE_NOBLOCK, false); AlterTableSimpleDesc desc = new AlterTableSimpleDesc( return; mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj)); isArchived = ArchiveUtils.isArchived(part); Path tabPath = tblObj.getPath(); Path partPath = part.getDataLocation(); newTblPartLoc = new Path(tabPath.toUri().getScheme(), tabPath.toUri() .getAuthority(), partPath.toUri().getPath()); inputFormatClass = tblObj.getInputFormatClass(); bucketCols = tblObj.getBucketCols(); Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false)); mergeTask.addDependentTask(moveTsk); moveTsk.addDependentTask(statTask);
JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null; if (out != null) { out.print(indentString(indent)); out.print(task.getId()); if ((task.getParentTasks() == null || task.getParentTasks().isEmpty())) { if (task.isRootTask()) { if (out != null) { out.print(" is a root stage"); StringBuilder s = new StringBuilder(); first = true; for (Task<?> parent : task.getParentTasks()) { if (!first) { s.append(", "); s.append(parent.getId()); Task<?> currBackupTask = task.getBackupTask(); if (currBackupTask != null) { if (out != null) { out.print(" has a backup stage: "); out.print(currBackupTask.getId()); json.put("BACKUP STAGE", currBackupTask.getId()); s.append(con.getId()); if (out != null) {
pathToAliases.put(new Path("path1"), new ArrayList<String>(Arrays.asList("alias1", "alias2"))); pathToAliases.put(new Path("path2"), new ArrayList<String>(Arrays.asList("alias3"))); new LinkedHashMap<Task<? extends Serializable>, Set<String>>(); taskToAliases.put(task1, new HashSet<String>(Arrays.asList("alias2"))); taskToAliases.put(task2, new HashSet<String>(Arrays.asList("alias3"))); ctx.setAliasToKnownSize(aliasToKnownSize); HiveConf conf = new HiveConf(); conf.setLongVar(HiveConf.ConfVars.HIVESMALLTABLESFILESIZE, 4096); Assert.assertEquals("alias3", resolved.getId()); conf.setLongVar(HiveConf.ConfVars.HIVESMALLTABLESFILESIZE, 65536); Assert.assertEquals("alias3", resolved.getId()); conf.setLongVar(HiveConf.ConfVars.HIVESMALLTABLESFILESIZE, 2048);
if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); if (ts.tableHandle.isNonNative()) { throw new SemanticException(ErrorMsg.LOAD_INTO_NON_NATIVE.getMsg()); if (TableType.MANAGED_TABLE.equals(ts.tableHandle.getTableType()) && conf.getBoolVar(HiveConf.ConfVars.HIVECHECKFILEFORMAT)) { ensureFileFormatsMatch(ts, files, fromURI); inputs.add(toReadEntity(new Path(fromURI))); partSpec = new LinkedHashMap<String, String>(); outputs.add(new WriteEntity(ts.tableHandle, (isOverWrite ? WriteEntity.WriteType.INSERT_OVERWRITE : LoadTableDesc loadTableWork = new LoadTableDesc(new Path(fromURI), Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING, writeId); loadTableWork.setStmtId(stmtId); if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { BasicStatsWork basicStatsWork = new BasicStatsWork(loadTableWork); basicStatsWork.setNoStatsAggregator(true); childTask.addDependentTask(statTask);
if (tblObj.isPartitioned()) { if (partSpec == null) { throw new SemanticException("source table " + tableName newTblPartLoc = new Path(tabPath.toUri().getScheme(), tabPath.toUri() .getAuthority(), partPath.toUri().getPath()); ddlWork.setNeedLock(true); Task<? extends Serializable> mergeTask = TaskFactory.get(ddlWork, conf); TableDesc tblDesc = Utilities.getTableDesc(tblObj); Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc); mergeDesc.setOutputDir(queryTmpdir); Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf); mergeTask.addDependentTask(moveTsk); if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { StatsWork statDesc; if (oldTblPartLoc.equals(newTblPartLoc)) { statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); Task<? extends Serializable> statTask = TaskFactory.get(statDesc, conf); moveTsk.addDependentTask(statTask);
boolean noName = StringUtils.isEmpty(conf.get(MRJobConfig.JOB_NAME)); this.driverCxt = driverCxt; // for canceling the query (should be bound to session?) SessionState.get().setMapRedStats(new LinkedHashMap<>()); assert tsk.getParentTasks() == null || tsk.getParentTasks().isEmpty(); driverCxt.addToRunnable(tsk); tsk.updateTaskMetrics(metrics); queryDisplay.setTaskResult(tskRun.getTask().getId(), tskRun.getTaskResult()); Task<? extends Serializable> backupTask = tsk.getAndInitBackupTask(); if (backupTask != null) { setErrorMsgAndDetail(exitVal, result.getTaskError(), tsk); errorMessage + Strings.nullToEmpty(tsk.getDiagnosticsMessage()), result.getTaskError()); SQLState = "08S01"; SessionState.get().getHiveHistory().setTaskProperty(queryId, tsk.getId(), if (tsk.getChildTasks() != null) { for (Task<? extends Serializable> child : tsk.getChildTasks()) { if (DriverContext.isLaunchable(child)) { driverCxt.addToRunnable(child);
return; pCtx.getFetchTask().getWork().setHiveServerQuery(SessionState.get().isHiveServerQuery()); TableDesc resultTab = pCtx.getFetchTask().getTblDesc(); if (SessionState.get().isHiveServerQuery()) { conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER,ThriftFormatter.class.getName()); } else { String formatterName = conf.get(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER); if (formatterName == null || formatterName.isEmpty()) { conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, DefaultFetchFormatter.class.getName()); Map<String, StatsTask> map = new LinkedHashMap<>(); if (isCStats) { if (rootTasks == null || rootTasks.size() != 1 || pCtx.getTopOps() == null StatsTask tsk = (StatsTask) genTableStats(pCtx, pCtx.getTopOps().values() .iterator().next(), root, outputs); root.addDependentTask(tsk); map.put(extractTableFullName(tsk), tsk); } catch (HiveException e) { tsk.addDependentTask(cStatsTask); Task<? extends Serializable> materializedViewTask = TaskFactory.get(materializedViewDesc, conf); for (Task<? extends Serializable> task : leafTasks) { task.addDependentTask(materializedViewTask);
if (tuple.isExternalTable) { URI fromURI = EximUtil.getValidatedURI(context.hiveConf, context.location); Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath()); isLocationSet = true; isExternal = true; FileSystem fs = FileSystem.get(fromURI, context.hiveConf); try { MetaData rv = EximUtil.readMetaData(fs, new Path(fromPath, EximUtil.METADATA_NAME)); Table table = new Table(rv.getTable()); parsedLocation = ReplExternalTables .externalTableLocation(context.hiveConf, table.getSd().getLocation()); } catch (IOException e) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); if (openTxnTask != null && !importTasks.isEmpty()) { for (Task<? extends Serializable> t : importTasks) { openTxnTask.addDependentTask(t);
new SparkWork(parseContext.getConf().getVar(HiveConf.ConfVars.HIVEQUERYID)); newWork.add(childWork); copyWorkGraph(currentWork, newWork, childWork); newWork.connect(mapWork, childWork, originEdge); String streamDesc = taskTmpDir.toUri().toString(); if (GenMapRedUtils.needsTagging((ReduceWork) childWork)) { Operator<? extends OperatorDesc> childReducer = ((ReduceWork) childWork).getReducer(); while (mapWork.getAliasToWork().get(streamDesc) != null) { streamDesc = origStreamDesc.concat(String.valueOf(++pos)); Task<? extends Serializable> childTask = childTasks.get(0); currentTask.removeDependentTask(childTask); newTask.addDependentTask(childTask); newTask.setFetchSource(currentTask.isFetchSource());
parentTask.addDependentTask(childTask); mapCurrCtx.put(tableScanOp, new GenMapRedCtx(childTask, null)); String streamDesc = taskTmpDir.toUri().toString(); MapredWork cplan = (MapredWork) childTask.getWork(); while (cplan.getMapWork().getAliasToWork().get(streamDesc) != null) { streamDesc = origStreamDesc.concat(String.valueOf(++pos));
String actualTblName = msg.getTables().get(idx); String actualDBName = msg.getDatabases().get(idx); String completeName = Table.getCompleteName(actualDBName, actualTblName); Path location = HiveUtils.getDumpPath(new Path(context.location), actualDBName, actualTblName); tblName = context.isTableNameEmpty() ? actualTblName : context.tableName; currentContext.setLocation(location.toUri().toString()); updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); context.log.debug("Added Commit txn task : {}", commitTxnTask.getId()); DAGTraversal.traverse(tasks, new AddDependencyToLeaves(commitTxnTask)); return tasks;
private static Task<?> loadTable(URI fromURI, Table table, boolean replace, Path tgtPath, ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x) { Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME); Path tmpPath = x.getCtx().getExternalTmpPath(tgtPath); Task<?> copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, x.getConf()); LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath, Utilities.getTableDesc(table), new TreeMap<String, String>(), replace); Task<?> loadTableTask = TaskFactory.get(new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false), x.getConf()); copyTask.addDependentTask(loadTableTask); x.getTasks().add(copyTask); return loadTableTask; }