try { VectorMapJoinFastTableContainer tableContainer = new VectorMapJoinFastTableContainer(mapJoinDesc, hconf, -1); tableContainer.setSerde(keyContext, valueContext); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { value.readFields(in); tableContainer.putRow(key, value); tableContainer.setKey(folder.toString()); tableContainer.seal(); return tableContainer; } catch (IOException e) {
public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, long estimatedKeyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; keyCountAdj = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT); threshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD); loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); this.estimatedKeyCount = estimatedKeyCount; int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, estimatedKeyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); vectorMapJoinFastHashTable = createHashTable(newThreshold); }
new VectorMapJoinFastTableContainer(desc, hconf, keyCount); vectorMapJoinFastTableContainer.putRow( null, (BytesWritable) kvReader.getCurrentKey(), null, (BytesWritable) kvReader.getCurrentValue()); vectorMapJoinFastTableContainer.seal(); mapJoinTables[pos] = (MapJoinTableContainer) vectorMapJoinFastTableContainer;
case FAST: mapJoinTableContainer = new VectorMapJoinFastTableContainer( mapJoinDesc, testDesc.hiveConf, testData.smallTableKeyHashMap.size()); break;
try { VectorMapJoinFastTableContainer tableContainer = new VectorMapJoinFastTableContainer(mapJoinDesc, hconf, -1); tableContainer.setSerde(keyContext, valueContext); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { value.readFields(in); tableContainer.putRow(key, value); tableContainer.seal(); return tableContainer; } catch (IOException e) {
public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, long estimatedKeyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; keyCountAdj = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT); threshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD); loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); this.estimatedKeyCount = estimatedKeyCount; // LOG.info("VectorMapJoinFastTableContainer load keyCountAdj " + keyCountAdj); // LOG.info("VectorMapJoinFastTableContainer load threshold " + threshold); // LOG.info("VectorMapJoinFastTableContainer load loadFactor " + loadFactor); // LOG.info("VectorMapJoinFastTableContainer load wbSize " + wbSize); int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, estimatedKeyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); vectorMapJoinFastHashTable = createHashTable(newThreshold); }
new VectorMapJoinFastTableContainer(desc, hconf, keyCount); vectorMapJoinFastTableContainer.setSerde(null, null); // No SerDes here. while (kvReader.next()) { vectorMapJoinFastTableContainer.putRow((BytesWritable)kvReader.getCurrentKey(), (BytesWritable)kvReader.getCurrentValue()); numEntries++; if (doMemCheck && (numEntries % memoryMonitorInfo.getMemoryCheckInterval() == 0)) { final long estMemUsage = vectorMapJoinFastTableContainer.getEstimatedMemorySize(); if (estMemUsage > effectiveThreshold) { String msg = "Hash table loading exceeded memory limits for input: " + inputName + vectorMapJoinFastTableContainer.seal(); mapJoinTables[pos] = vectorMapJoinFastTableContainer; if (doMemCheck) { LOG.info("Finished loading hash table for input: {} cacheKey: {} numEntries: {} " + "estimatedMemoryUsage: {}", inputName, cacheKey, numEntries, vectorMapJoinFastTableContainer.getEstimatedMemorySize()); } else { LOG.info("Finished loading hash table for input: {} cacheKey: {} numEntries: {}", inputName, cacheKey,
public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, long keyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; keyCountAdj = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT); threshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD); loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); this.keyCount = keyCount; // LOG.info("VectorMapJoinFastTableContainer load keyCountAdj " + keyCountAdj); // LOG.info("VectorMapJoinFastTableContainer load threshold " + threshold); // LOG.info("VectorMapJoinFastTableContainer load loadFactor " + loadFactor); // LOG.info("VectorMapJoinFastTableContainer load wbSize " + wbSize); int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); VectorMapJoinFastHashTable = createHashTable(newThreshold); }
new VectorMapJoinFastTableContainer(desc, hconf, keyCount); vectorMapJoinFastTableContainer.setSerde(null, null); // No SerDes here. while (kvReader.next()) { vectorMapJoinFastTableContainer.putRow((BytesWritable)kvReader.getCurrentKey(), (BytesWritable)kvReader.getCurrentValue()); vectorMapJoinFastTableContainer.seal(); mapJoinTables[pos] = (MapJoinTableContainer) vectorMapJoinFastTableContainer;
desc.setVectorDesc(vectorDesc); Configuration hconf = new HiveConf(); VectorMapJoinFastTableContainer container = new VectorMapJoinFastTableContainer(desc, hconf, keyCount); container.setSerde(null, null); value = new BytesWritable(outp.getData(), outp.getLength()); container.putRow(key, value); dataSize += 8; dataSize += 8; Long executionEstimate = container.getEstimatedMemorySize(); Long compilerEstimate = null;