@Override public void closeOp(boolean abort) throws HiveException { super.closeOp(abort); // clean up LOG.debug("Cleaning up the operator state"); for (int pos = 0; pos < order.length; pos++) { if (pos != posBigTable) { fetchDone[pos] = false; } foundNextKeyGroup[pos] = false; } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @SuppressWarnings("unchecked") AbstractMapJoinOperator<? extends MapJoinDesc> mjOp = (AbstractMapJoinOperator<? extends MapJoinDesc>) nd; MapJoinDesc mjDesc = mjOp.getConf(); String bigTablAlias = mjDesc.getBigTableAlias(); if ( bigTablAlias == null ) { Operator<? extends OperatorDesc> parent = null; for(Operator<? extends OperatorDesc> op : mjOp.getParentOperators() ) { if ( op instanceof TableScanOperator ) { parent = op; } } if ( parent != null) { TableScanDesc tDesc = ((TableScanOperator)parent).getConf(); bigTablAlias = tDesc.getAlias(); } } bigTablAlias = bigTablAlias == null ? "?" : bigTablAlias; List<ExprNodeDesc> joinExprs = mjDesc.getKeys().values().iterator().next(); if ( joinExprs.size() == 0 ) { warnings.add( String.format("Map Join %s[bigTable=%s] in task '%s' is a cross product", mjOp.toString(), bigTablAlias, taskName)); } return null; } }
@Override public void initializeLocalWork(Configuration hconf) throws HiveException { Operator<? extends OperatorDesc> parent = null; for (Operator<? extends OperatorDesc> parentOp : parentOperators) { if (parentOp != null) { parent = parentOp; break; } } if (parent == null) { throw new HiveException("No valid parents."); } if (parentOperators.size() == 1) { Map<Integer, DummyStoreOperator> dummyOps = ((TezContext) (MapredContext.get())).getDummyOpsMap(); for (Entry<Integer, DummyStoreOperator> connectOp : dummyOps.entrySet()) { if (connectOp.getValue().getChildOperators() == null || connectOp.getValue().getChildOperators().isEmpty()) { parentOperators.add(connectOp.getKey(), connectOp.getValue()); connectOp.getValue().getChildOperators().add(this); } } } super.initializeLocalWork(hconf); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { AbstractMapJoinOperator<MapJoinDesc> op = (AbstractMapJoinOperator<MapJoinDesc>) nd; pruneJoinOperator(ctx, op, op.getConf(), op.getColumnExprMap(), op .getConf().getRetainList(), true); return null; } }
GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos)); Task<? extends Serializable> currTask = mapredCtx.getCurrTask(); MapredWork currPlan = (MapredWork) currTask.getWork(); boolean local = pos != mapJoin.getConf().getPosBigTable(); if (oldTask == null) { assert currPlan.getReduceWork() == null;
public static <T> Set<T> findOperatorsUpstreamJoinAccounted(Operator<?> start, Class<T> clazz, Set<T> found) { if (clazz.isInstance(start)) { found.add((T) start); } int onlyIncludeIndex = -1; if (start instanceof AbstractMapJoinOperator) { AbstractMapJoinOperator mapJoinOp = (AbstractMapJoinOperator) start; MapJoinDesc desc = (MapJoinDesc) mapJoinOp.getConf(); onlyIncludeIndex = desc.getPosBigTable(); } if (start.getParentOperators() != null) { int i = 0; for (Operator<?> parent : start.getParentOperators()) { if (onlyIncludeIndex >= 0) { if (onlyIncludeIndex == i) { findOperatorsUpstreamJoinAccounted(parent, clazz, found); } } else { findOperatorsUpstreamJoinAccounted(parent, clazz, found); } i++; } } return found; }
@SuppressWarnings("unchecked") @Override public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); firstFetchHappened = false; fetchInputAtClose = getFetchInputAtCloseList();
public static int getPositionParent(AbstractMapJoinOperator<? extends MapJoinDesc> op, Stack<Node> stack) { int pos = 0; int size = stack.size(); assert size >= 2 && stack.get(size - 1) == op; Operator<? extends OperatorDesc> parent = (Operator<? extends OperatorDesc>) stack.get(size - 2); List<Operator<? extends OperatorDesc>> parOp = op.getParentOperators(); pos = parOp.indexOf(parent); assert pos < parOp.size(); return pos; }
@Override @SuppressWarnings("unchecked") protected void initializeOp(Configuration hconf) throws HiveException { if (conf.getGenJoinKeys()) { int tagLen = conf.getTagLength(); joinKeys = new List[tagLen]; JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), NOTSKIPBIGTABLE, hconf); joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys, inputObjInspectors,NOTSKIPBIGTABLE, tagLen); } super.initializeOp(hconf); numMapRowsRead = 0; // all other tables are small, and are cached in the hash table posBigTable = (byte) conf.getPosBigTable(); emptyList = new RowContainer<List<Object>>(1, hconf, reporter); RowContainer<List<Object>> bigPosRC = JoinUtil.getRowContainer(hconf, rowContainerStandardObjectInspectors[posBigTable], posBigTable, joinCacheSize,spillTableDesc, conf, !hasFilter(posBigTable), reporter); storage[posBigTable] = bigPosRC; }
@Override protected List<ObjectInspector> getValueObjectInspectors( byte alias, List<ObjectInspector>[] aliasToObjectInspectors) { int[] valueIndex = conf.getValueIndex(alias); if (valueIndex == null) { return super.getValueObjectInspectors(alias, aliasToObjectInspectors); } List<ObjectInspector> inspectors = aliasToObjectInspectors[alias]; int bigPos = conf.getPosBigTable(); Converter[] converters = new Converter[valueIndex.length]; List<ObjectInspector> valueOI = new ArrayList<ObjectInspector>(); for (int i = 0; i < valueIndex.length; i++) { if (valueIndex[i] >= 0 && !joinKeysObjectInspectors[bigPos].isEmpty()) { if (conf.getNoOuterJoin()) { valueOI.add(joinKeysObjectInspectors[bigPos].get(valueIndex[i])); } else { // It is an outer join. We are going to add the inspector from the // inner side, but the key value will come from the outer side, so // we need to create a converter from inputOI to outputOI. valueOI.add(inspectors.get(i)); converters[i] = ObjectInspectorConverters.getConverter( joinKeysObjectInspectors[bigPos].get(valueIndex[i]), inspectors.get(i)); } } else { valueOI.add(inspectors.get(i)); } } unwrapContainer[alias] = new UnwrapRowContainer(alias, valueIndex, converters, hasFilter(alias)); return valueOI; }
@Override public void close(boolean abort) throws HiveException { joinFinalLeftData(); // Do this WITHOUT checking for parents super.close(abort); }
GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos)); Task<? extends Serializable> currTask = mapredCtx.getCurrTask(); MapredWork currPlan = (MapredWork) currTask.getWork(); boolean local = pos != mapJoin.getConf().getPosBigTable(); if (oldTask == null) { assert currPlan.getReduceWork() == null;
public static <T> Set<T> findOperatorsUpstreamJoinAccounted(Operator<?> start, Class<T> clazz, Set<T> found) { if (clazz.isInstance(start)) { found.add((T) start); } int onlyIncludeIndex = -1; if (start instanceof AbstractMapJoinOperator) { AbstractMapJoinOperator mapJoinOp = (AbstractMapJoinOperator) start; MapJoinDesc desc = (MapJoinDesc) mapJoinOp.getConf(); onlyIncludeIndex = desc.getPosBigTable(); } if (start.getParentOperators() != null) { int i = 0; for (Operator<?> parent : start.getParentOperators()) { if (onlyIncludeIndex >= 0) { if (onlyIncludeIndex == i) { findOperatorsUpstream(parent, clazz, found); } } else { findOperatorsUpstream(parent, clazz, found); } i++; } } return found; }
@SuppressWarnings("unchecked") @Override public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); firstFetchHappened = false; fetchInputAtClose = getFetchInputAtCloseList();
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { AbstractMapJoinOperator<MapJoinDesc> op = (AbstractMapJoinOperator<MapJoinDesc>) nd; pruneJoinOperator(ctx, op, op.getConf(), op.getColumnExprMap(), op .getConf().getRetainList(), true); return null; } }
public static int getPositionParent(AbstractMapJoinOperator<? extends MapJoinDesc> op, Stack<Node> stack) { int pos = 0; int size = stack.size(); assert size >= 2 && stack.get(size - 1) == op; Operator<? extends OperatorDesc> parent = (Operator<? extends OperatorDesc>) stack.get(size - 2); List<Operator<? extends OperatorDesc>> parOp = op.getParentOperators(); pos = parOp.indexOf(parent); assert pos < parOp.size(); return pos; }
@Override @SuppressWarnings("unchecked") protected void initializeOp(Configuration hconf) throws HiveException { if (conf.getGenJoinKeys()) { int tagLen = conf.getTagLength(); joinKeys = new List[tagLen]; JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), NOTSKIPBIGTABLE, hconf); joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys, inputObjInspectors,NOTSKIPBIGTABLE, tagLen); } super.initializeOp(hconf); numMapRowsRead = 0; // all other tables are small, and are cached in the hash table posBigTable = (byte) conf.getPosBigTable(); emptyList = new RowContainer<List<Object>>(1, hconf, reporter); RowContainer<List<Object>> bigPosRC = JoinUtil.getRowContainer(hconf, rowContainerStandardObjectInspectors[posBigTable], posBigTable, joinCacheSize,spillTableDesc, conf, !hasFilter(posBigTable), reporter); storage[posBigTable] = bigPosRC; }
@Override protected List<ObjectInspector> getValueObjectInspectors( byte alias, List<ObjectInspector>[] aliasToObjectInspectors) { int[] valueIndex = conf.getValueIndex(alias); if (valueIndex == null) { return super.getValueObjectInspectors(alias, aliasToObjectInspectors); } List<ObjectInspector> inspectors = aliasToObjectInspectors[alias]; int bigPos = conf.getPosBigTable(); Converter[] converters = new Converter[valueIndex.length]; List<ObjectInspector> valueOI = new ArrayList<ObjectInspector>(); for (int i = 0; i < valueIndex.length; i++) { if (valueIndex[i] >= 0 && !joinKeysObjectInspectors[bigPos].isEmpty()) { if (conf.getNoOuterJoin()) { valueOI.add(joinKeysObjectInspectors[bigPos].get(valueIndex[i])); } else { // It is an outer join. We are going to add the inspector from the // inner side, but the key value will come from the outer side, so // we need to create a converter from inputOI to outputOI. valueOI.add(inspectors.get(i)); converters[i] = ObjectInspectorConverters.getConverter( joinKeysObjectInspectors[bigPos].get(valueIndex[i]), inspectors.get(i)); } } else { valueOI.add(inspectors.get(i)); } } unwrapContainer[alias] = new UnwrapRowContainer(alias, valueIndex, converters, hasFilter(alias)); return valueOI; }
@Override public void close(boolean abort) throws HiveException { joinFinalLeftData(); // Do this WITHOUT checking for parents super.close(abort); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @SuppressWarnings("unchecked") AbstractMapJoinOperator<? extends MapJoinDesc> mjOp = (AbstractMapJoinOperator<? extends MapJoinDesc>) nd; MapJoinDesc mjDesc = mjOp.getConf(); String bigTablAlias = mjDesc.getBigTableAlias(); if ( bigTablAlias == null ) { Operator<? extends OperatorDesc> parent = null; for(Operator<? extends OperatorDesc> op : mjOp.getParentOperators() ) { if ( op instanceof TableScanOperator ) { parent = op; } } if ( parent != null) { TableScanDesc tDesc = ((TableScanOperator)parent).getConf(); bigTablAlias = tDesc.getAlias(); } } bigTablAlias = bigTablAlias == null ? "?" : bigTablAlias; List<ExprNodeDesc> joinExprs = mjDesc.getKeys().values().iterator().next(); if ( joinExprs.size() == 0 ) { warnings.add( String.format("Map Join %s[bigTable=%s] in task '%s' is a cross product", mjOp.toString(), bigTablAlias, taskName)); } return null; } }