/** * Transform the query tree. For each join, check which reduce sink will * output the biggest result (based on STREAMTABLE hints) and give it the * biggest tag so that it gets streamed. * * @param pactx * current parse context */ @Override public ParseContext transform(ParseContext pactx) throws SemanticException { Set<String> bigTables = getBigTables(pactx); cache.clear(); for (JoinOperator joinOp : pactx.getJoinOps()) { reorder(joinOp, bigTables); } return pactx; } }
/** * Transform the query tree. For each join, check which reduce sink will * output the biggest result (based on STREAMTABLE hints) and give it the * biggest tag so that it gets streamed. * * @param pactx * current parse context */ @Override public ParseContext transform(ParseContext pactx) throws SemanticException { Set<String> bigTables = getBigTables(pactx); cache.clear(); for (JoinOperator joinOp : pactx.getJoinOps()) { reorder(joinOp, bigTables); } return pactx; } }
/** * Find all big tables from STREAMTABLE hints. * * @param joinCtx * The join context * @return Set of all big tables */ private Set<String> getBigTables(ParseContext joinCtx) { Set<String> bigTables = new HashSet<String>(); for (JoinOperator joinOp : joinCtx.getJoinOps()) { if (joinOp.getConf().getStreamAliases() != null) { bigTables.addAll(joinOp.getConf().getStreamAliases()); } } return bigTables; }
/** * Find all big tables from STREAMTABLE hints. * * @param joinCtx * The join context * @return Set of all big tables */ private Set<String> getBigTables(ParseContext joinCtx) { Set<String> bigTables = new HashSet<String>(); for (JoinOperator joinOp : joinCtx.getJoinOps()) { if (joinOp.getConf().getStreamAliases() != null) { bigTables.addAll(joinOp.getConf().getStreamAliases()); } } return bigTables; }
/** * Change existing references in the context to point from child to parent operator. * @param cSEL child operator (to be removed, and merged into parent) * @param pSEL parent operator */ private void fixContextReferences(SelectOperator cSEL, SelectOperator pSEL) { Collection<Map<String, Operator<? extends OperatorDesc>>> mapsAliasToOpInfo = new ArrayList<Map<String, Operator<? extends OperatorDesc>>>(); for (JoinOperator joinOp : pctx.getJoinOps()) { if (joinOp.getConf().getAliasToOpInfo() != null) { mapsAliasToOpInfo.add(joinOp.getConf().getAliasToOpInfo()); } } for (MapJoinOperator mapJoinOp : pctx.getMapJoinOps()) { if (mapJoinOp.getConf().getAliasToOpInfo() != null) { mapsAliasToOpInfo.add(mapJoinOp.getConf().getAliasToOpInfo()); } } for (Map<String, Operator<? extends OperatorDesc>> aliasToOpInfo : mapsAliasToOpInfo) { for (Map.Entry<String, Operator<? extends OperatorDesc>> entry : aliasToOpInfo.entrySet()) { if (entry.getValue() == cSEL) { aliasToOpInfo.put(entry.getKey(), pSEL); } } } } }
/** * Change existing references in the context to point from child to parent operator. * @param cSEL child operator (to be removed, and merged into parent) * @param pSEL parent operator */ private void fixContextReferences(SelectOperator cSEL, SelectOperator pSEL) { Collection<Map<String, Operator<? extends OperatorDesc>>> mapsAliasToOpInfo = new ArrayList<Map<String, Operator<? extends OperatorDesc>>>(); for (JoinOperator joinOp : pctx.getJoinOps()) { if (joinOp.getConf().getAliasToOpInfo() != null) { mapsAliasToOpInfo.add(joinOp.getConf().getAliasToOpInfo()); } } for (MapJoinOperator mapJoinOp : pctx.getMapJoinOps()) { if (mapJoinOp.getConf().getAliasToOpInfo() != null) { mapsAliasToOpInfo.add(mapJoinOp.getConf().getAliasToOpInfo()); } } for (Map<String, Operator<? extends OperatorDesc>> aliasToOpInfo : mapsAliasToOpInfo) { for (Map.Entry<String, Operator<? extends OperatorDesc>> entry : aliasToOpInfo.entrySet()) { if (entry.getValue() == cSEL) { aliasToOpInfo.put(entry.getKey(), pSEL); } } } } }
protected boolean checkConvertJoinToSMBJoin( JoinOperator joinOperator, SortBucketJoinProcCtx smbJoinContext) throws SemanticException { if (!this.pGraphContext.getJoinOps().contains(joinOperator)) { return false; } String[] srcs = joinOperator.getConf().getBaseSrc(); // All the tables/partitions columns should be sorted in the same order // For example, if tables A and B are being joined on columns c1, c2 and c3 // which are the sorted and bucketed columns. The join would work, as long // c1, c2 and c3 are sorted in the same order. List<Order> sortColumnsFirstTable = new ArrayList<Order>(); for (int pos = 0; pos < srcs.length; pos++) { if (!isEligibleForBucketSortMergeJoin(smbJoinContext, smbJoinContext.getKeyExprMap().get((byte) pos), joinOperator.getConf().getAliasToOpInfo(), srcs, pos, sortColumnsFirstTable)) { return false; } } smbJoinContext.setSrcs(srcs); return true; }
protected boolean checkConvertJoinToSMBJoin( JoinOperator joinOperator, SortBucketJoinProcCtx smbJoinContext) throws SemanticException { if (!this.pGraphContext.getJoinOps().contains(joinOperator)) { return false; } String[] srcs = joinOperator.getConf().getBaseSrc(); // All the tables/partitions columns should be sorted in the same order // For example, if tables A and B are being joined on columns c1, c2 and c3 // which are the sorted and bucketed columns. The join would work, as long // c1, c2 and c3 are sorted in the same order. List<Order> sortColumnsFirstTable = new ArrayList<Order>(); for (int pos = 0; pos < srcs.length; pos++) { if (!isEligibleForBucketSortMergeJoin(smbJoinContext, smbJoinContext.getKeyExprMap().get((byte) pos), joinOperator.getConf().getAliasToOpInfo(), srcs, pos, sortColumnsFirstTable)) { return false; } } smbJoinContext.setSrcs(srcs); return true; }
pctx.getJoinOps().add(joinOp);
pctx.getJoinOps().add(joinOp);
if (pactx.getJoinOps() != null) { Set<JoinOperator> joinMap = new HashSet<JoinOperator>(); Set<MapJoinOperator> mapJoinMap = pactx.getMapJoinOps(); Iterator<JoinOperator> joinCtxIter = pactx.getJoinOps().iterator(); while (joinCtxIter.hasNext()) { JoinOperator joinOp = joinCtxIter.next();
for (JoinOperator joinOp: pCtx.getJoinOps()) { boolean isAbleToGuess = true; boolean mayConvert = false;
protected MapJoinOperator convertJoinToBucketMapJoin( JoinOperator joinOp, SortBucketJoinProcCtx joinContext) throws SemanticException { MapJoinOperator mapJoinOp = new MapJoinProcessor().convertMapJoin( pGraphContext.getConf(), joinOp, joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), joinOp.getConf().getMapAliases(), joinContext.getBigTablePosition(), false, false); if (mapJoinOp == null) { return null; } // Remove the join operator from the query join context // Data structures coming from QBJoinTree mapJoinOp.getConf().setQBJoinTreeProps(joinOp.getConf()); // pGraphContext.getMapJoinOps().add(mapJoinOp); pGraphContext.getJoinOps().remove(joinOp); convertMapJoinToBucketMapJoin(mapJoinOp, joinContext); return mapJoinOp; }
String id = null; if (reducerOp instanceof JoinOperator) { if (parseCtx.getJoinOps().contains(reducerOp)) { id = ((JoinOperator)reducerOp).getConf().getId();
String id = null; if (childReducer instanceof JoinOperator) { if (parseContext.getJoinOps().contains(childReducer)) { id = ((JoinOperator)childReducer).getConf().getId();
protected MapJoinOperator convertJoinToBucketMapJoin( JoinOperator joinOp, SortBucketJoinProcCtx joinContext) throws SemanticException { MapJoinOperator mapJoinOp = new MapJoinProcessor().convertMapJoin( pGraphContext.getConf(), joinOp, joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), joinOp.getConf().getMapAliases(), joinContext.getBigTablePosition(), false, false); // Remove the join operator from the query join context // Data structures coming from QBJoinTree mapJoinOp.getConf().setQBJoinTreeProps(joinOp.getConf()); // pGraphContext.getMapJoinOps().add(mapJoinOp); pGraphContext.getJoinOps().remove(joinOp); convertMapJoinToBucketMapJoin(mapJoinOp, joinContext); return mapJoinOp; }
if (!this.pGraphContext.getJoinOps().contains(joinOp)) { return false;
if (!this.pGraphContext.getJoinOps().contains(joinOp)) { return false;
/** * Create a clone of the parse context */ public ParseContext getParseContext(ParseContext pCtx, List<Task<? extends Serializable>> rootTasks) { ParseContext clone = new ParseContext(queryState, pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(), pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(), pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getColumnStatsAutoGatherContexts(), pCtx.getContext(), pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(), pCtx.getListMapJoinOpsNoReducer(), pCtx.getPrunedPartitions(), pCtx.getTabNameToTabObject(), pCtx.getOpToSamplePruner(), pCtx.getGlobalLimitCtx(), pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks, pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(), pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(), pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getCreateViewDesc(), pCtx.getQueryProperties(), pCtx.getViewProjectToTableSchema(), pCtx.getAcidSinks()); clone.setFetchTask(pCtx.getFetchTask()); clone.setLineageInfo(pCtx.getLineageInfo()); clone.setMapJoinOps(pCtx.getMapJoinOps()); clone.setRsToRuntimeValuesInfoMap(pCtx.getRsToRuntimeValuesInfoMap()); clone.setRsOpToTsOpMap(pCtx.getRsOpToTsOpMap()); return clone; }
/** * Create a clone of the parse context */ public ParseContext getParseContext(ParseContext pCtx, List<Task<? extends Serializable>> rootTasks) { ParseContext clone = new ParseContext(queryState, pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(), pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(), pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getColumnStatsAutoGatherContexts(), pCtx.getContext(), pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(), pCtx.getListMapJoinOpsNoReducer(), pCtx.getPrunedPartitions(), pCtx.getTabNameToTabObject(), pCtx.getOpToSamplePruner(), pCtx.getGlobalLimitCtx(), pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks, pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(), pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(), pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getCreateViewDesc(), pCtx.getMaterializedViewUpdateDesc(), pCtx.getQueryProperties(), pCtx.getViewProjectToTableSchema(), pCtx.getAcidSinks()); clone.setFetchTask(pCtx.getFetchTask()); clone.setLineageInfo(pCtx.getLineageInfo()); clone.setMapJoinOps(pCtx.getMapJoinOps()); clone.setRsToRuntimeValuesInfoMap(pCtx.getRsToRuntimeValuesInfoMap()); clone.setRsToSemiJoinBranchInfo(pCtx.getRsToSemiJoinBranchInfo()); clone.setColExprToGBMap(pCtx.getColExprToGBMap()); clone.setSemiJoinHints(pCtx.getSemiJoinHints()); return clone; }