/** * Initialize the visitor to use the QueryDefDeserializer Use the order * defined in QueryDefWalker to visit the QueryDef * * @param hiveConf * @throws HiveException */ protected void reconstructQueryDef(Configuration hiveConf) throws HiveException { PTFDeserializer dS = new PTFDeserializer(conf, (StructObjectInspector)inputObjInspectors[0], hiveConf); dS.initializePTFChain(conf.getFuncDef()); }
private PTFInvocation setupChain() { Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>(); PTFInputDef iDef = conf.getFuncDef(); while (iDef instanceof PartitionedTableFunctionDef) { fnDefs.push((PartitionedTableFunctionDef) iDef); iDef = ((PartitionedTableFunctionDef) iDef).getInput(); } PTFInvocation curr = null, first = null; while(!fnDefs.isEmpty()) { PartitionedTableFunctionDef currFn = fnDefs.pop(); curr = new PTFInvocation(curr, currFn.getTFunction()); if ( first == null ) { first = curr; } } return first; }
/** * Initialize the visitor to use the QueryDefDeserializer Use the order * defined in QueryDefWalker to visit the QueryDef * * @param hiveConf * @throws HiveException */ protected void reconstructQueryDef(Configuration hiveConf) throws HiveException { PTFDeserializer dS = new PTFDeserializer(conf, (StructObjectInspector)inputObjInspectors[0], hiveConf); dS.initializePTFChain(conf.getFuncDef()); }
private PTFInvocation setupChain() { Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>(); PTFInputDef iDef = conf.getFuncDef(); while (iDef instanceof PartitionedTableFunctionDef) { fnDefs.push((PartitionedTableFunctionDef) iDef); iDef = ((PartitionedTableFunctionDef) iDef).getInput(); } PTFInvocation curr = null, first = null; while(!fnDefs.isEmpty()) { PartitionedTableFunctionDef currFn = fnDefs.pop(); curr = new PTFInvocation(curr, currFn.getTFunction()); if ( first == null ) { first = curr; } } return first; }
public static void alterOutputOIForStreaming(PTFDesc ptfDesc) { PartitionedTableFunctionDef tDef = ptfDesc.getFuncDef(); TableFunctionEvaluator tEval = tDef.getTFunction(); if ( tEval.canIterateOutput() ) { tDef.getOutputShape().setOI(tEval.getOutputOI()); } }
public static void alterOutputOIForStreaming(PTFDesc ptfDesc) { PartitionedTableFunctionDef tDef = ptfDesc.getFuncDef(); TableFunctionEvaluator tEval = tDef.getTFunction(); if ( tEval.canIterateOutput() ) { tDef.getOutputShape().setOI(tEval.getOutputOI()); } }
List<String> partitionKeys = new ArrayList<>(); PartitionDef partition = ptfOp.getConf().getFuncDef().getPartition(); if (partition != null && partition.getExpressions() != null) {
PTFDesc ptfDesc = translator.translate(wSpec, semanticAnalyzer, hiveConf, rr, unparseTranslator); RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
WindowTableFunctionDef wTFn = (WindowTableFunctionDef) conf.getFuncDef(); List<Integer> rFnIdxs = rankingFunctions(wTFn);
WindowTableFunctionDef wTFn = (WindowTableFunctionDef) conf.getFuncDef(); List<Integer> rFnIdxs = rankingFunctions(wTFn);
PTFDesc ptfDesc = translator.translate(wSpec, semanticAnalyzer, hiveConf, rr, unparseTranslator); RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr(); input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()),
RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr(); input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()),
PTFTranslator translator = new PTFTranslator(); PTFDesc ptfDesc = translator.translate(wSpec, this, conf, rr, unparseTranslator); RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr(); input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()), input), ptfOpRR);
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
PTFTranslator translator = new PTFTranslator(); PTFDesc ptfDesc = translator.translate(wSpec, this, conf, rr, unparseTranslator); RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr(); input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()), input), ptfOpRR);
@Override protected void initializeOp(Configuration jobConf) throws HiveException { super.initializeOp(jobConf); hiveConf = jobConf; isMapOperator = conf.isMapSide(); currentKeys = null; reconstructQueryDef(hiveConf); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); outputObjInspector = tDef.getRawInputShape().getOI(); } else { outputObjInspector = conf.getFuncDef().getOutputShape().getOI(); } setupKeysWrapper(inputObjInspectors[0]); ptfInvocation = setupChain(); ptfInvocation.initializeStreaming(jobConf, isMapOperator); firstMapRow = true; }
@Override protected void initializeOp(Configuration jobConf) throws HiveException { super.initializeOp(jobConf); hiveConf = jobConf; isMapOperator = conf.isMapSide(); currentKeys = null; reconstructQueryDef(hiveConf); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); outputObjInspector = tDef.getRawInputShape().getOI(); } else { outputObjInspector = conf.getFuncDef().getOutputShape().getOI(); } setupKeysWrapper(inputObjInspectors[0]); ptfInvocation = setupChain(); ptfInvocation.initializeStreaming(jobConf, isMapOperator); firstMapRow = true; }
throws HiveException { PartitionedTableFunctionDef funcDef = ptfDesc.getFuncDef();