/** * Clones only the operator. The children and parent are set * to null. * @return Cloned operator * @throws CloneNotSupportedException */ @SuppressWarnings("unchecked") public Operator<? extends OperatorDesc> cloneOp() throws CloneNotSupportedException { T descClone = (T) conf.clone(); Operator<? extends OperatorDesc> ret = OperatorFactory.getAndMakeChild(cContext, descClone, getSchema()); return ret; }
private void setStatistics(Operator<? extends OperatorDesc> origin, Operator<? extends OperatorDesc> clone) { clone.getConf().setStatistics(origin.getConf().getStatistics()); clone.getConf().setTraits(origin.getConf().getTraits()); if (origin.getChildOperators().size() == clone.getChildOperators().size()) { for (int i = 0; i < clone.getChildOperators().size(); i++) { setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i)); } } } }
totalFreeMemory = conf.getMaxMemoryAvailable() - memoryUsedPerExecutor;
Statistics stats = conf.getStatistics(); if (stats == null && op.getParentOperators() != null) { op.getConf().setStatistics(stats);
public Statistics getStatistics() { return (conf == null) ? null : conf.getStatistics(); }
public OpTraits getOpTraits() { return (conf == null) ? null : conf.getTraits(); }
public void setOpTraits(OpTraits metaInfo) { LOG.debug("Setting traits ({}) on: {}", metaInfo, this); if (conf != null) { conf.setTraits(metaInfo); } else { LOG.warn("Cannot set traits when there is no descriptor: {}", this); } }
public void setStatistics(Statistics stats) { LOG.debug("Setting stats ({}) on: {}", stats, this); if (conf != null) { conf.setStatistics(stats); } else { LOG.warn("Cannot set stats when there is no descriptor: {}", this); } }
public static void setMemoryAvailable(final List<Operator<? extends OperatorDesc>> operators, final long memoryAvailableToTask) { if (operators == null) { return; } for (Operator<? extends OperatorDesc> op : operators) { if (op.getConf() != null) { op.getConf().setMaxMemoryAvailable(memoryAvailableToTask); } if (op.getChildOperators() != null && !op.getChildOperators().isEmpty()) { setMemoryAvailable(op.getChildOperators(), memoryAvailableToTask); } } }
/** * Operator specific close routine. Operators which inherents this class * should overwrite this funtion for their specific cleanup routine. */ protected void closeOp(boolean abort) throws HiveException { if (conf != null && conf.getRuntimeStatsTmpDir() != null) { publishRunTimeStats(); } runTimeNumRows = 0; }
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
Statistics stats = conf.getStatistics(); if (stats == null) { if (op.getParentOperators() != null) { parentStats, op.getColumnExprMap(), op.getSchema()); stats.addToColumnStats(colStats); op.getConf().setStatistics(stats);
public Statistics getStatistics() { if (conf != null) { return conf.getStatistics(); } return null; }
public OpTraits getOpTraits() { if (conf != null) { return conf.getTraits(); } return null; }
public void setOpTraits(OpTraits metaInfo) { if (isLogDebugEnabled) { LOG.debug("Setting traits (" + metaInfo + ") on " + this); } if (conf != null) { conf.setTraits(metaInfo); } else { LOG.warn("Cannot set traits when there's no descriptor: " + this); } }
public void setStatistics(Statistics stats) { if (isLogDebugEnabled) { LOG.debug("Setting stats (" + stats + ") on " + this); } if (conf != null) { conf.setStatistics(stats); } else { LOG.warn("Cannot set stats when there's no descriptor: " + this); } }
public static void setMemoryAvailable(final List<Operator<? extends OperatorDesc>> operators, final long memoryAvailableToTask) { if (operators == null) { return; } for (Operator<? extends OperatorDesc> op : operators) { if (op.getConf() != null) { op.getConf().setMaxMemoryAvailable(memoryAvailableToTask); } if (op.getChildOperators() != null && !op.getChildOperators().isEmpty()) { setMemoryAvailable(op.getChildOperators(), memoryAvailableToTask); } } }
if (conf != null && conf.getRuntimeStatsTmpDir() != null) { publishRunTimeStats();
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
private void setStatistics(Operator<? extends OperatorDesc> origin, Operator<? extends OperatorDesc> clone) { clone.getConf().setStatistics(origin.getConf().getStatistics()); clone.getConf().setTraits(origin.getConf().getTraits()); if (origin.getChildOperators().size() == clone.getChildOperators().size()) { for (int i = 0; i < clone.getChildOperators().size(); i++) { setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i)); } } } }