@Override public VectorMapJoinDesc clone() { VectorMapJoinDesc clone = new VectorMapJoinDesc(); clone.hashTableImplementationType = this.hashTableImplementationType; clone.hashTableKind = this.hashTableKind; clone.hashTableKeyType = this.hashTableKeyType; clone.vectorMapJoinVariation = this.vectorMapJoinVariation; clone.minMaxEnabled = this.minMaxEnabled; if (vectorMapJoinInfo != null) { throw new RuntimeException("Cloning VectorMapJoinInfo not supported"); } return clone; }
public MapJoinOperatorExplainVectorization(MapJoinDesc mapJoinDesc, VectorMapJoinDesc vectorMapJoinDesc) { // VectorMapJoinOperator is not native vectorized. super( vectorMapJoinDesc, vectorMapJoinDesc.getHashTableImplementationType() != HashTableImplementationType.NONE); this.mapJoinDesc = mapJoinDesc; this.vectorMapJoinDesc = vectorMapJoinDesc; vectorMapJoinInfo = (vectorMapJoinDesc != null ? vectorMapJoinDesc.getVectorMapJoinInfo() : null); }
boolean enabled = vectorMapJoinDesc.getIsVectorizationMapJoinNativeEnabled(); String engine = vectorMapJoinDesc.getEngine(); String engineInSupportedCondName = HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + boolean isFastHashTableEnabled = vectorMapJoinDesc.getIsFastHashTableEnabled(); vectorMapJoinDesc.getUseOptimizedTable(), HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE.varname)); conditionList.add( conditionList.add( new VectorizationCondition( vectorMapJoinDesc.getOneMapJoinCondition(), "One MapJoin Condition")); conditionList.add( new VectorizationCondition( !vectorMapJoinDesc.getHasNullSafes(), "No nullsafe")); conditionList.add( new VectorizationCondition( vectorMapJoinDesc.getSmallTableExprVectorizes(), "Small table vectorizes")); if (!mapJoinDesc.isNoOuterJoin()) { conditionList.add( new VectorizationCondition( !vectorMapJoinDesc.getOuterJoinHasNoKeys(), "Outer Join has keys"));
public MapJoinOperatorExplainVectorization(MapJoinDesc mapJoinDesc, VectorDesc vectorDesc) { // VectorMapJoinOperator is not native vectorized. super(vectorDesc, ((VectorMapJoinDesc) vectorDesc).hashTableImplementationType() != HashTableImplementationType.NONE); this.mapJoinDesc = mapJoinDesc; vectorMapJoinDesc = (VectorMapJoinDesc) vectorDesc; vectorMapJoinInfo = vectorMapJoinDesc.getVectorMapJoinInfo(); }
vectorMapJoinInfo.setSlimmedBigTableKeyExpressions(slimmedBigTableKeyExpressions); vectorDesc.setAllBigTableKeyExpressions(allBigTableKeyExpressions); vectorMapJoinInfo.setSlimmedBigTableValueExpressions(slimmedBigTableValueExpressions); vectorDesc.setAllBigTableValueExpressions(allBigTableValueExpressions); vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo); vectorDesc.setUseOptimizedTable(useOptimizedTable); vectorDesc.setIsVectorizationMapJoinNativeEnabled(isVectorizationMapJoinNativeEnabled); vectorDesc.setEngine(engine); vectorDesc.setOneMapJoinCondition(oneMapJoinCondition); vectorDesc.setHasNullSafes(hasNullSafes); vectorDesc.setSmallTableExprVectorizes(smallTableExprVectorizes); vectorDesc.setOuterJoinHasNoKeys(outerJoinHasNoKeys); vectorDesc.setIsFastHashTableEnabled(isFastHashTableEnabled); vectorDesc.setIsHybridHashJoin(isHybridHashJoin); vectorDesc.setSupportsKeyTypes(supportsKeyTypes); if (!supportsKeyTypes) { vectorDesc.setNotSupportedKeyTypes(new ArrayList(notSupportedKeyTypes)); vectorDesc.setSupportsValueTypes(supportsValueTypes); if (!supportsValueTypes) { vectorDesc.setNotSupportedValueTypes(new ArrayList(notSupportedValueTypes));
public static VectorMapJoinDesc createVectorMapJoinDesc(MapJoinTestDescription testDesc) { VectorMapJoinDesc vectorDesc = new VectorMapJoinDesc(); vectorDesc.setHashTableImplementationType(HashTableImplementationType.FAST); HashTableKind hashTableKind; switch (testDesc.vectorMapJoinVariation) { throw new RuntimeException("unknown operator variation " + testDesc.vectorMapJoinVariation); vectorDesc.setHashTableKind(hashTableKind); vectorDesc.setHashTableKeyType(hashTableKeyType); vectorDesc.setVectorMapJoinVariation(testDesc.vectorMapJoinVariation); vectorDesc.setMinMaxEnabled(false); vectorMapJoinInfo.setSlimmedBigTableKeyExpressions(null); vectorDesc.setAllBigTableKeyExpressions(null); vectorMapJoinInfo.setSlimmedBigTableValueExpressions(null); vectorDesc.setAllBigTableValueExpressions(null); }; vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo);
Class<? extends Operator<?>> opClass = null; VectorMapJoinInfo vectorMapJoinInfo = vectorDesc.getVectorMapJoinInfo(); VectorMapJoinVariation vectorMapJoinVariation = null; if (vectorDesc.getIsFastHashTableEnabled()) { hashTableImplementationType = HashTableImplementationType.FAST; } else { HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_MINMAX_ENABLED); vectorDesc.setHashTableImplementationType(hashTableImplementationType); vectorDesc.setHashTableKind(hashTableKind); vectorDesc.setHashTableKeyType(hashTableKeyType); vectorDesc.setVectorMapJoinVariation(vectorMapJoinVariation); if (vectorMapJoinVariation == VectorMapJoinVariation.FULL_OUTER) { vectorDesc.setIsFullOuter(true); vectorDesc.setMinMaxEnabled(minMaxEnabled); vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo);
VectorMapJoinDesc vectorDesc = new VectorMapJoinDesc(); desc.setVectorDesc(vectorDesc); vectorDesc.setUseOptimizedTable(useOptimizedTable); vectorDesc.setIsVectorizationMapJoinNativeEnabled(isVectorizationMapJoinNativeEnabled); vectorDesc.setEngine(engine); vectorDesc.setOneMapJoinCondition(oneMapJoinCondition); vectorDesc.setHasNullSafes(hasNullSafes); vectorDesc.setSmallTableExprVectorizes(smallTableExprVectorizes); vectorDesc.setIsFastHashTableEnabled(isFastHashTableEnabled); vectorDesc.setIsHybridHashJoin(isHybridHashJoin); vectorDesc.setSupportsKeyTypes(supportsKeyTypes); if (!supportsKeyTypes) { vectorDesc.setNotSupportedKeyTypes(new ArrayList(notSupportedKeyTypes));
OperatorVariation operatorVariation = OperatorVariation.NONE; if (vectorDesc.getIsFastHashTableEnabled()) { hashTableImplementationType = HashTableImplementationType.FAST; } else { HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_MINMAX_ENABLED); vectorDesc.setHashTableImplementationType(hashTableImplementationType); vectorDesc.setHashTableKind(hashTableKind); vectorDesc.setHashTableKeyType(hashTableKeyType); vectorDesc.setOperatorVariation(operatorVariation); vectorDesc.setMinMaxEnabled(minMaxEnabled); vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo);
boolean enabled = vectorMapJoinDesc.getIsVectorizationMapJoinNativeEnabled(); String engine = vectorMapJoinDesc.getEngine(); String engineInSupportedCondName = HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " + vectorizableMapJoinNativeEngines; boolean engineInSupported = vectorizableMapJoinNativeEngines.contains(engine); boolean isFastHashTableEnabled = vectorMapJoinDesc.getIsFastHashTableEnabled(); vectorMapJoinDesc.getUseOptimizedTable(), HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE.varname)); conditionList.add( conditionList.add( new VectorizationCondition( vectorMapJoinDesc.getOneMapJoinCondition(), "One MapJoin Condition")); conditionList.add( new VectorizationCondition( !vectorMapJoinDesc.getHasNullSafes(), "No nullsafe")); conditionList.add( new VectorizationCondition( vectorMapJoinDesc.getSmallTableExprVectorizes(), "Small table vectorizes")); conditionList.add( new VectorizationCondition( !vectorMapJoinDesc.getIsHybridHashJoin(),
private void runEstimationCheck(HashTableKeyType l) throws SerDeException, IOException, HiveException { MapJoinDesc desc = new MapJoinDesc(); VectorMapJoinDesc vectorDesc = new VectorMapJoinDesc(); vectorDesc.setHashTableKeyType(l); vectorDesc.setIsFastHashTableEnabled(true); vectorDesc.setHashTableImplementationType(HashTableImplementationType.FAST); vectorDesc.setHashTableKind(HashTableKind.HASH_MAP); desc.setVectorDesc(vectorDesc); Configuration hconf = new HiveConf();
vectorDesc.setHashTableImplementationType(hashTableImplementationType); vectorDesc.setHashTableKind(hashTableKind); vectorDesc.setHashTableKeyType(hashTableKeyType); vectorDesc.setMinMaxEnabled(minMaxEnabled); return vectorOp;
mapJoinDesc.setVectorDesc(vectorDesc); vectorDesc.setHashTableImplementationType(hashTableImplementationType); VectorMapJoinInfo vectorMapJoinInfo = vectorDesc.getVectorMapJoinInfo(); switch (vectorDesc.getHashTableImplementationType()) { case OPTIMIZED: mapJoinTableContainer = break; default: throw new RuntimeException("Unexpected hash table implementation type " + vectorDesc.getHashTableImplementationType());
VectorMapJoinDesc vectorMapJoinDesc = new VectorMapJoinDesc(); boolean specialize = canSpecializeMapJoin(op, desc, isTezOrSpark, vContext, vectorMapJoinDesc); VectorMapJoinInfo vectorMapJoinInfo = vectorMapJoinDesc.getVectorMapJoinInfo(); if (usesVectorUDFAdaptor(vectorMapJoinDesc.getAllBigTableKeyExpressions())) { vectorTaskColumnInfo.setUsesVectorUDFAdaptor(true); if (usesVectorUDFAdaptor(vectorMapJoinDesc.getAllBigTableValueExpressions())) { vectorTaskColumnInfo.setUsesVectorUDFAdaptor(true);
VectorMapJoinDesc vectorMapJoinDesc = new VectorMapJoinDesc(); vectorMapJoinDesc.setAllBigTableKeyExpressions(allBigTableKeyExpressions); vectorMapJoinDesc.setAllBigTableValueExpressions(allBigTableValueExpressions);
this.conf = desc; this.vectorDesc = (VectorMapJoinDesc) vectorDesc; vectorMapJoinInfo = this.vectorDesc.getVectorMapJoinInfo(); Preconditions.checkState(vectorMapJoinInfo != null); isOuterJoin = !desc.getNoOuterJoin(); vectorMapJoinVariation = this.vectorDesc.getVectorMapJoinVariation(); hashTableKind = this.vectorDesc.getHashTableKind(); hashTableKeyType = this.vectorDesc.getHashTableKeyType();
this.conf = desc; vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); vectorMapJoinInfo = vectorDesc.getVectorMapJoinInfo(); Preconditions.checkState(vectorMapJoinInfo != null);
@Override public VectorMapJoinDesc clone() { VectorMapJoinDesc clone = new VectorMapJoinDesc(); clone.hashTableImplementationType = this.hashTableImplementationType; clone.hashTableKind = this.hashTableKind; clone.hashTableKeyType = this.hashTableKeyType; clone.operatorVariation = this.operatorVariation; clone.minMaxEnabled = this.minMaxEnabled; if (vectorMapJoinInfo != null) { throw new RuntimeException("Cloning VectorMapJoinInfo not supported"); } return clone; }
public MapJoinDesc(MapJoinDesc clone) { super(clone); vectorDesc = new VectorMapJoinDesc(clone.vectorDesc); this.keys = clone.keys; this.keyTblDesc = clone.keyTblDesc; this.valueTblDescs = clone.valueTblDescs; this.posBigTable = clone.posBigTable; this.valueIndices = clone.valueIndices; this.retainList = clone.retainList; this.bigTableAlias = clone.bigTableAlias; this.aliasBucketFileNameMapping = clone.aliasBucketFileNameMapping; this.bigTableBucketNumMapping = clone.bigTableBucketNumMapping; this.bigTablePartSpecToFileMapping = clone.bigTablePartSpecToFileMapping; this.dumpFilePrefix = clone.dumpFilePrefix; this.parentToInput = clone.parentToInput; this.parentKeyCounts = clone.parentKeyCounts; this.parentDataSizes = clone.parentDataSizes; this.isBucketMapJoin = clone.isBucketMapJoin; this.isHybridHashJoin = clone.isHybridHashJoin; }
public MapJoinDesc() { vectorDesc = new VectorMapJoinDesc(); bigTableBucketNumMapping = new LinkedHashMap<String, Integer>(); }