if (oi instanceof WritableStringObjectInspector) { WritableStringObjectInspector woi = (WritableStringObjectInspector) oi; return JavaDataModel.get().lengthForStringOfLength( woi.getPrimitiveWritableObject(value).getLength()); } else if (oi instanceof WritableBinaryObjectInspector) { WritableBinaryObjectInspector woi = (WritableBinaryObjectInspector) oi; return JavaDataModel.get().lengthForByteArrayOfSize( woi.getPrimitiveWritableObject(value).getLength()); } else if (oi instanceof WritableBooleanObjectInspector) { return JavaDataModel.get().primitive1(); } else if (oi instanceof WritableByteObjectInspector) { return JavaDataModel.get().primitive1(); } else if (oi instanceof WritableDateObjectInspector) { return JavaDataModel.get().lengthOfDate(); } else if (oi instanceof WritableDoubleObjectInspector) { return JavaDataModel.get().primitive2(); } else if (oi instanceof WritableFloatObjectInspector) { return JavaDataModel.get().primitive1(); } else if (oi instanceof WritableHiveDecimalObjectInspector) { return JavaDataModel.get().lengthOfDecimal(); } else if (oi instanceof WritableIntObjectInspector) { return JavaDataModel.get().primitive1(); } else if (oi instanceof WritableLongObjectInspector) { return JavaDataModel.get().primitive2(); } else if (oi instanceof WritableShortObjectInspector) { return JavaDataModel.get().primitive1(); } else if (oi instanceof WritableTimestampObjectInspector || oi instanceof WritableTimestampLocalTZObjectInspector) { return JavaDataModel.get().lengthOfTimestamp();
@InterfaceAudience.LimitedPrivate(value = {"Hive" }) static int lengthFor(JavaDataModel model, Integer numVector) { int length = model.object(); length += model.primitive1() * 2; // two int length += model.primitive2(); // one double length += model.lengthForRandom() * 2; // two Random if (numVector == null) { numVector = 16; // HiveConf hive.stats.ndv.error default produces 16 vectors } if (numVector > 0) { length += model.array() * 3; // three array length += model.primitive1() * numVector * 2; // two int array length += (model.object() + model.array() + model.primitive1() + model.primitive2()) * numVector; // bitset array } return length; }
public int lengthFor(JavaDataModel model) { int length = model.object(); length += model.primitive1() * 2; // two int int numBins = getNumBins(); if (numBins > 0) { length += model.arrayList(); // List<Coord> // Coord holds two doubles length += numBins * (model.object() + model.primitive2() * 2); } length += model.lengthForRandom(); // Random return length; } }
@Override public long getAggregationBufferFixedSize() { JavaDataModel model = JavaDataModel.get(); return JavaDataModel.alignUp( model.object() + model.primitive2()*3+ model.primitive1(), model.memoryAlign()); }
@Override public long getAggregationBufferFixedSize() { JavaDataModel model = JavaDataModel.get(); return JavaDataModel.alignUp( model.object() + model.ref()+ model.primitive1()*2, model.memoryAlign()); }
public static int calculateEstimatedMemorySize(int count) { // FUTURE: Partitioning not included yet. final int longMatchFlagsSize = (count + Long.SIZE - 1) / Long.SIZE; int size = 0; JavaDataModel jdm = JavaDataModel.get(); size += jdm.lengthForLongArrayOfSize(longMatchFlagsSize); size += jdm.primitive1(); size += (2 * jdm.object()); return size; } }
|| colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) { int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase); result += JavaDataModel.get().lengthForStringOfLength(avgColLen); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase); result += JavaDataModel.get().lengthForByteArrayOfSize(avgColLen); } else { result += getAvgColLenOfFixedLengthTypes(colTypeLowerCase); result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length, conf); } else { result += JavaDataModel.get().lengthForObjectArrayOfSize(length); result += JavaDataModel.get().arrayList() + (listEntries * elemSize); result += JavaDataModel.get().hashMap(mapEntries); result += JavaDataModel.get().object(); result += soi.getAllStructFieldRefs().size() * JavaDataModel.get().ref(); for (StructField field : soi.getAllStructFieldRefs()) { result += getSizeOfComplexTypes(conf, field.getFieldObjectInspector()); result += JavaDataModel.get().object(); result += uoi.getObjectInspectors().size() * JavaDataModel.get().primitive1(); for (ObjectInspector foi : uoi.getObjectInspectors()) { result += getSizeOfComplexTypes(conf, foi);
|| colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) { int acl = (int) Math.round(cs.getAvgColLen()); sizeOf = JavaDataModel.get().lengthForStringOfLength(acl); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { int acl = (int) Math.round(cs.getAvgColLen()); sizeOf = JavaDataModel.get().lengthForByteArrayOfSize(acl); } else if (colTypeLowerCase.equals(serdeConstants.TIMESTAMP_TYPE_NAME) || colTypeLowerCase.equals(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME)) { sizeOf = JavaDataModel.get().lengthOfTimestamp(); } else if (colTypeLowerCase.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) { sizeOf = JavaDataModel.get().lengthOfDecimal(); } else if (colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) { sizeOf = JavaDataModel.get().lengthOfDate(); } else { sizeOf = cs.getAvgColLen();
|| colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) || colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) { return JavaDataModel.get().lengthForIntArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) { return JavaDataModel.get().lengthForDoubleArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME) || colTypeLowerCase.equals("long")) { return JavaDataModel.get().lengthForLongArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { return JavaDataModel.get().lengthForByteArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.BOOLEAN_TYPE_NAME)) { return JavaDataModel.get().lengthForBooleanArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.TIMESTAMP_TYPE_NAME) || colTypeLowerCase.equals(serdeConstants.DATETIME_TYPE_NAME) || colTypeLowerCase.equals(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME) || colTypeLowerCase.equals(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME)) { return JavaDataModel.get().lengthForTimestampArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) { return JavaDataModel.get().lengthForDateArrayOfSize(length); } else if (colTypeLowerCase.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) { return JavaDataModel.get().lengthForDecimalArrayOfSize(length); } else if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) { int configVarLen = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAX_VARIABLE_LENGTH); int siz = JavaDataModel.get().lengthForStringOfLength(configVarLen); return JavaDataModel.get().lengthForPrimitiveArrayOfSize(siz, length); } else {
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = 0; size += vectorMapJoinFastHashTable.getEstimatedMemorySize(); size += (4 * jdm.primitive1()); size += (2 * jdm.object()); size += (jdm.primitive2()); return size; }
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = 0; size += keySerDe == null ? 0 : jdm.object(); size += valSerDe == null ? 0 : jdm.object(); size += keySoi == null ? 0 : DEFAULT_OBJECT_INSPECTOR_MEMORY_SIZE; size += valSoi == null ? 0 : DEFAULT_OBJECT_INSPECTOR_MEMORY_SIZE; size += keyOis == null ? 0 : jdm.arrayList() + keyOis.size() * DEFAULT_OBJECT_INSPECTOR_MEMORY_SIZE; size += valOis == null ? 0 : jdm.arrayList() + valOis.size() * DEFAULT_OBJECT_INSPECTOR_MEMORY_SIZE; size += keyObjs == null ? 0 : jdm.array() + keyObjs.length * jdm.object(); size += valObjs == null ? 0 : jdm.array() + valObjs.length * jdm.object(); size += jdm.primitive1(); return size; } }
@Override public int estimate() { JavaDataModel model = JavaDataModel.get(); return (int) (model.primitive1() * 2 + model.primitive2() * 4 + model.lengthFor(columnType) + ((numDV == null) ? lengthFor(model, null) : numDV.lengthFor(model))); } };
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = 0; size += writeBuffers == null ? 0 : jdm.arrayList() + (writeBuffers.size() * jdm.lengthForByteArrayOfSize(wbSize)); size += (3 * jdm.primitive2()); size += writePos == null ? 0 : writePos.getEstimatedMemorySize(); size += unsafeReadPos == null ? 0 : unsafeReadPos.getEstimatedMemorySize(); return size; }
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = 0; size += writeBuffers.getEstimatedMemorySize(); size += jdm.lengthForLongArrayOfSize(refs.length); // 11 primitive1 fields, 2 refs above with alignment size += JavaDataModel.alignUp(15 * jdm.primitive1(), jdm.memoryAlign()); return size; }
public int lengthForRandom() { // boolean + double + AtomicLong return object() + primitive1() + primitive2() + object() + primitive2(); }
public int lengthForStringOfLength(int strLen) { return object() + primitive1() * 3 + array() + strLen; } }
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long size = 0; size += (4 * jdm.object()); size += jdm.primitive1(); return size; } }
@Override public int estimate() { JavaDataModel model = JavaDataModel.get(); return super.estimate() + model.lengthOfDecimal() * 2; }
@Override public int estimate() { JavaDataModel model = JavaDataModel.get(); return (int) (model.primitive2() * 3 + model.lengthFor(columnType)); } };
@Override public long getEstimatedMemorySize() { JavaDataModel jdm = JavaDataModel.get(); long memSize = buffer == null ? 0 : jdm.lengthForByteArrayOfSize(buffer.length); memSize += (2 * jdm.primitive1()); return memSize; } public void set(Position pos) {