Refine search![Refine arrow](//d16xvuom8mr9m8.cloudfront.net/code/public/images/refine-arrow.226bef887e7b1c430d6d498717d9d561.svg)
public int getPosition(String internalName) { int pos = -1; for (ColumnInfo var : rowSchema.getSignature()) { ++pos; if (var.getInternalName().equals(internalName)) { return pos; } } return -1; }
private static ArrayList<ColumnInfo> createColInfos(Operator<?> input) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); for (ColumnInfo ci : input.getSchema().getSignature()) { cInfoLst.add(new ColumnInfo(ci)); } return cInfoLst; }
private static ArrayList<ColumnInfo> createColInfosSubset(Operator<?> input, List<String> keepColNames) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); int pos = 0; for (ColumnInfo ci : input.getSchema().getSignature()) { if (pos < keepColNames.size() && ci.getInternalName().equals(keepColNames.get(pos))) { cInfoLst.add(new ColumnInfo(ci)); pos++; } } return cInfoLst; }
/** * Match column in skewed column list and record position. * The position will be used in {@link FileSinkOperator} generateListBucketingDirName(). * Note that skewed column name matches skewed value in order. * * @param rowSch */ public void processRowSkewedIndex(RowSchema rowSch) { if ((this.skewedColNames != null) && (this.skewedColNames.size() > 0) && (rowSch != null) && (rowSch.getSignature() != null) && (rowSch.getSignature().size() > 0)) { List<ColumnInfo> cols = rowSch.getSignature(); int hitNo = 0; for (int i = 0; i < cols.size(); i++) { int index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); if (index > -1) { hitNo++; SkewedColumnPositionPair pair = new SkewedColumnPositionPair(i, index); rowSkewedIndex.add(pair); } } assert (hitNo == this.skewedColNames.size()) : "RowSchema doesn't have all skewed columns." + "Skewed column: " + this.skewedColNames.toString() + ". Rowschema has columns: " + cols; } }
/** * Set the lineage information for the associated directory. * * @param dir The directory containing the query results. * @param dc The associated data container. * @param cols The list of columns. */ public synchronized void setLineage(Path dir, DataContainer dc, List<FieldSchema> cols) { // First lookup the file sink operator from the load work. Operator<?> op = dirToFop.get(dir.toUri().toString()); // Go over the associated fields and look up the dependencies // by position in the row schema of the filesink operator. if (op == null) { return; } List<ColumnInfo> signature = op.getSchema().getSignature(); int i = 0; for (FieldSchema fs : cols) { linfo.putDependency(dc, fs, index.getDependency(op, signature.get(i++))); } }
if (colList != null) { for (int i = 0; i < colList.size(); i++) { ExprNodeDesc newCol = foldExpr(colList.get(i), constants, cppCtx, op, 0, false); if (!(colList.get(i) instanceof ExprNodeConstantDesc) && newCol instanceof ExprNodeConstantDesc) { String colName = colList.get(i).getExprString(); if (HiveConf.getPositionFromInternalName(colName) == -1) { ColumnInfo colInfo = op.getSchema().getSignature().get(i); if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) { constants.put(colInfo, newCol);
private static void pruneOperator(NodeProcessorCtx ctx, Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { // the pruning needs to preserve the order of columns in the input schema RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>(); RowSchema oldRS = op.getSchema(); for(ColumnInfo i : oldRS.getSignature()) { if (lookupColumn(cols, i.getInternalName()) != null) { rs.add(i); } } op.getSchema().setSignature(rs); } }
/** * Match column in skewed column list and record position. * The position will be used in {@link FileSinkOperator} generateListBucketingDirName(). * Note that skewed column name matches skewed value in order. * * @param rowSch */ public void processRowSkewedIndex(RowSchema rowSch) { if ((this.skewedColNames != null) && (this.skewedColNames.size() > 0) && (rowSch != null) && (rowSch.getSignature() != null) && (rowSch.getSignature().size() > 0)) { List<ColumnInfo> cols = rowSch.getSignature(); int hitNo = 0; for (int i = 0; i < cols.size(); i++) { int index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); if (index > -1) { hitNo++; SkewedColumnPositionPair pair = new SkewedColumnPositionPair(i, index); rowSkewedIndex.add(pair); } } assert (hitNo == this.skewedColNames.size()) : "RowSchema doesn't have all skewed columns." + "Skewed column: " + this.skewedColNames.toString() + ". Rowschema has columns: " + cols; } }
private static ArrayList<ColumnInfo> createColInfosSubset(Operator<?> input, List<String> keepColNames) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); int pos = 0; for (ColumnInfo ci : input.getSchema().getSignature()) { if (pos < keepColNames.size() && ci.getInternalName().equals(keepColNames.get(pos))) { cInfoLst.add(new ColumnInfo(ci)); pos++; } } return cInfoLst; }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSinkForJoin(int initialPos, List<String> outputColumnNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp, String tabAlias) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); for (int i = 0; i < index.length; i++) { ColumnInfo info = new ColumnInfo(inputOp.getSchema().getSignature().get(i)); String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, tabAlias, info.getIsVirtualCol()); columnDescriptors.put(outputColumnNames.get(initialPos + i), desc); } return columnDescriptors; }
/** * Inserts a filter below the table scan operator. Construct the filter * from the filter expression provided. * @param tableScanOp the table scan operators * @param filterExpr the filter expression */ private void insertFilterOnTop( TableScanOperator tableScanOp, ExprNodeDesc filterExpr) { // Get the top operator and it's child, all operators have a single parent Operator<? extends OperatorDesc> currChild = tableScanOp.getChildOperators().get(0); // Create the filter Operator and update the parents and children appropriately tableScanOp.setChildOperators(null); currChild.setParentOperators(null); Operator<FilterDesc> filter = OperatorFactory.getAndMakeChild( new FilterDesc(filterExpr, false), new RowSchema(tableScanOp.getSchema().getSignature()), tableScanOp); OperatorFactory.makeChild(filter, currChild); }
/** * The pruning needs to preserve the order of columns in the input schema * @param op * @param cols * @return * @throws SemanticException */ private static List<FieldNode> preserveColumnOrder(Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<FieldNode> rs = new ArrayList<>(); ArrayList<ColumnInfo> inputCols = inputSchema.getSignature(); for (ColumnInfo i: inputCols) { FieldNode fn = lookupColumn(cols, i.getInternalName()); if (fn != null) { rs.add(fn); } } return rs; } else { return cols; } }
public int getPosition(String internalName) { int pos = -1; for (ColumnInfo var : rowSchema.getSignature()) { ++pos; if (var.getInternalName().equals(internalName)) { return pos; } } return -1; }
private static ArrayList<ColumnInfo> createColInfos(Operator<?> input) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); for (ColumnInfo ci : input.getSchema().getSignature()) { cInfoLst.add(new ColumnInfo(ci)); } return cInfoLst; }
static ASTNode buildSQJoinExpr(String sqAlias, RowResolver sqRR) { List<ColumnInfo> signature = sqRR.getRowSchema().getSignature(); ColumnInfo joinColumn = signature.get(0); String[] joinColName = sqRR.reverseLookup(joinColumn.getInternalName()); return createColRefAST(sqAlias, joinColName[1]); }
RowSchema dest = this.getParentOperators().get(0).getSchema(); if(orig.getSignature() == null && dest.getSignature() == null) { return true; if((orig.getSignature() == null && dest.getSignature() != null) || (orig.getSignature() != null && dest.getSignature() == null) ) { return false; if(orig.getSignature().size() != dest.getSignature().size() || orig.getSignature().size() != conf.getColList().size()) { return false; for(int i=0; i<orig.getSignature().size(); i++) { ColumnInfo origColumn = orig.getSignature().get(i); ColumnInfo destColumn = dest.getSignature().get(i); if(!(conf.getColList().get(i) instanceof ExprNodeColumnDesc)) { return false; ExprNodeColumnDesc col = (ExprNodeColumnDesc) conf.getColList().get(i); if(!col.getColumn().equals(origColumn.getInternalName())) { return false;
/** * Set the lineage information for the associated directory. * * @param dir The directory containing the query results. * @param dc The associated data container. * @param cols The list of columns. */ public void setLineage(Path dir, DataContainer dc, List<FieldSchema> cols) { // First lookup the file sink operator from the load work. Operator<?> op = dirToFop.get(dir); // Go over the associated fields and look up the dependencies // by position in the row schema of the filesink operator. if (op == null) { return; } List<ColumnInfo> signature = op.getSchema().getSignature(); int i = 0; for (FieldSchema fs : cols) { linfo.putDependency(dc, fs, index.getDependency(op, signature.get(i++))); } }
LOG.info("RS " + reduce.getIdentifier() + " oldColExprMap: " + oldMap); RowSchema oldRS = reduce.getSchema(); ArrayList<ColumnInfo> old_signature = oldRS.getSignature(); ArrayList<ColumnInfo> signature = new ArrayList<ColumnInfo>(old_signature); String outputCol = valueColNames.get(i); ExprNodeDesc outputColExpr = valueExprs.get(i); if (!retainFlags[i]) { ColumnInfo colInfo = oldRS.getColumnInfo(outputCol);
private Operator<? extends OperatorDesc> genOutputSelectForGroupBy( Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) { assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size()); Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator(); Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator(); List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>(); List<String> colName = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); while (pIter.hasNext()) { ColumnInfo pInfo = pIter.next(); ColumnInfo cInfo = cIter.next(); ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(), pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol()); columns.add(column); colName.add(cInfo.getInternalName()); columnExprMap.put(cInfo.getInternalName(), column); } return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp.getSchema().getSignature()), columnExprMap, parentOp); } }
private List<ColumnInfo> buildPrunedRS(List<FieldNode> prunedCols, RowSchema oldRS) throws SemanticException { ArrayList<ColumnInfo> sig = new ArrayList<ColumnInfo>(); HashSet<FieldNode> prunedColsSet = new HashSet<>(prunedCols); for (ColumnInfo cInfo : oldRS.getSignature()) { if (lookupColumn(prunedColsSet, cInfo.getInternalName()) != null) { sig.add(cInfo); } } return sig; }