public static ColumnStatisticsObj getTableColumnStatisticsObj( MTableColumnStatistics mStatsObj, boolean enableBitVector) { ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); statsObj.setColType(mStatsObj.getColType()); statsObj.setColName(mStatsObj.getColName()); String colType = mStatsObj.getColType().toLowerCase(); ColumnStatisticsData colStatsData = new ColumnStatisticsData(); boolStats.setNumFalses(mStatsObj.getNumFalses()); boolStats.setNumTrues(mStatsObj.getNumTrues()); boolStats.setNumNulls(mStatsObj.getNumNulls()); colStatsData.setBooleanStats(boolStats); } else if (colType.equals("string") || colType.startsWith("varchar") || colType.startsWith("char")) { StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); stringStats.setNumNulls(mStatsObj.getNumNulls()); stringStats.setAvgColLen(mStatsObj.getAvgColLen()); stringStats.setMaxColLen(mStatsObj.getMaxColLen()); stringStats.setNumDVs(mStatsObj.getNumDVs()); stringStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); colStatsData.setStringStats(stringStats); } else if (colType.equals("binary")) { BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); binaryStats.setNumNulls(mStatsObj.getNumNulls()); binaryStats.setAvgColLen(mStatsObj.getAvgColLen()); binaryStats.setMaxColLen(mStatsObj.getMaxColLen()); colStatsData.setBinaryStats(binaryStats); } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("timestamp")) {
public static void setFieldsIntoOldStats( MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStatsObj) { if (mStatsObj.getAvgColLen() != null) { oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); if (mStatsObj.getLongHighValue() != null) { oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); if (mStatsObj.getLongLowValue() != null) { oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue()); if (mStatsObj.getDoubleLowValue() != null) { oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue()); if (mStatsObj.getDoubleHighValue() != null) { oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue()); if (mStatsObj.getDecimalLowValue() != null) { oldStatsObj.setDecimalLowValue(mStatsObj.getDecimalLowValue()); if (mStatsObj.getDecimalHighValue() != null) { oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue()); if (mStatsObj.getMaxColLen() != null) { oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); if (mStatsObj.getNumDVs() != null) { oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); if (mStatsObj.getBitVector() != null) {
public static ColumnStatisticsDesc getTableColumnStatisticsDesc( MTableColumnStatistics mStatsObj) { ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); statsDesc.setIsTblLevel(true); statsDesc.setCatName(mStatsObj.getCatName()); statsDesc.setDbName(mStatsObj.getDbName()); statsDesc.setTableName(mStatsObj.getTableName()); statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed()); return statsDesc; }
private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStats) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { String colName = mStatsObj.getColName(); QueryWrapper queryWrapper = new QueryWrapper(); try { LOG.info("Updating table level column statistics for table={}" + " colName={}", Warehouse.getCatalogQualifiedTableName(table), colName); validateTableCols(table, Lists.newArrayList(colName)); if (oldStats != null) { StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector() == null) { // workaround for DN bug in persisting nulls in pg bytea column // instead set empty bit vector with header. mStatsObj.setBitVector(new byte[] {'H','L'}); } pm.makePersistent(mStatsObj); } } finally { queryWrapper.close(); } }
MTableColumnStatistics mColStats = new MTableColumnStatistics(); mColStats.setTable(table); mColStats.setDbName(statsDesc.getDbName()); mColStats.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName() : DEFAULT_CATALOG_NAME); mColStats.setTableName(statsDesc.getTableName()); mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); mColStats.setColName(statsObj.getColName()); mColStats.setColType(statsObj.getColType()); mColStats.setBooleanStats( boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, } else if (statsObj.getStatsData().isSetLongStats()) { LongColumnStatsData longStats = statsObj.getStatsData().getLongStats(); mColStats.setLongStats( longStats.isSetNumNulls() ? longStats.getNumNulls() : null, longStats.isSetNumDVs() ? longStats.getNumDVs() : null, } else if (statsObj.getStatsData().isSetDoubleStats()) { DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); mColStats.setDoubleStats( doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, String low = decimalStats.isSetLowValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getLowValue()) : null; String high = decimalStats.isSetHighValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getHighValue()) : null; mColStats.setDecimalStats( decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
private void writeMTableColumnStatistics(MTableColumnStatistics mStatsObj) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException String dbName = mStatsObj.getDbName(); String tableName = mStatsObj.getTableName(); String colName = mStatsObj.getColName(); MTable mTable = getMTable(mStatsObj.getDbName(), mStatsObj.getTableName()); boolean foundCol = false; if (mCol.getName().equals(mStatsObj.getColName().trim())) { foundCol = true; break; oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue()); oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed()); oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue()); oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue()); oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); oldStatsObj.setNumFalses(mStatsObj.getNumFalses()); oldStatsObj.setNumTrues(mStatsObj.getNumTrues()); oldStatsObj.setNumNulls(mStatsObj.getNumNulls()); } else { pm.makePersistent(mStatsObj);
public static ColumnStatisticsDesc getTableColumnStatisticsDesc( MTableColumnStatistics mStatsObj) { ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); statsDesc.setIsTblLevel(true); statsDesc.setDbName(mStatsObj.getDbName()); statsDesc.setTableName(mStatsObj.getTableName()); statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed()); return statsDesc; }
private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { String dbName = mStatsObj.getDbName(); String tableName = mStatsObj.getTableName(); String colName = mStatsObj.getColName(); LOG.info("Updating table level column statistics for db=" + dbName + " tableName=" + tableName + " colName=" + colName); validateTableCols(table, Lists.newArrayList(colName)); List<MTableColumnStatistics> oldStats = getMTableColumnStatistics(table, Lists.newArrayList(colName)); if (!oldStats.isEmpty()) { assert oldStats.size() == 1; StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); } else { pm.makePersistent(mStatsObj); } }
@Override protected ColumnStatistics getJdoResult( GetHelper<ColumnStatistics> ctx) throws MetaException { QueryWrapper queryWrapper = new QueryWrapper(); try { List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames, queryWrapper); if (mStats.isEmpty()) { return null; } // LastAnalyzed is stored per column, but thrift object has it per multiple columns. // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now. ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0)); List<ColumnStatisticsObj> statObjs = new ArrayList<>(mStats.size()); for (MTableColumnStatistics mStat : mStats) { if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) { desc.setLastAnalyzed(mStat.getLastAnalyzed()); } statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat, enableBitVector)); Deadline.checkTimeout(); } return new ColumnStatistics(desc, statObjs); } finally { queryWrapper.close(); } } }.run(true);
/** * Get table's column stats * * @param table * @param colNames * @return Map of column name and its stats * @throws NoSuchObjectException * @throws MetaException */ private Map<String, MTableColumnStatistics> getPartitionColStats(Table table, List<String> colNames) throws NoSuchObjectException, MetaException { Map<String, MTableColumnStatistics> statsMap = Maps.newHashMap(); QueryWrapper queryWrapper = new QueryWrapper(); try { List<MTableColumnStatistics> stats = getMTableColumnStatistics(table, colNames, queryWrapper); for(MTableColumnStatistics cStat : stats) { statsMap.put(cStat.getColName(), cStat); } } finally { queryWrapper.close(); } return statsMap; }
MTableColumnStatistics mColStats = new MTableColumnStatistics(); mColStats.setTable(table); mColStats.setDbName(statsDesc.getDbName()); mColStats.setTableName(statsDesc.getTableName()); mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); mColStats.setColName(statsObj.getColName()); mColStats.setColType(statsObj.getColType()); mColStats.setBooleanStats(boolStats.getNumTrues(), boolStats.getNumFalses(), boolStats.getNumNulls()); } else if (statsObj.getStatsData().isSetLongStats()) { LongColumnStatsData longStats = statsObj.getStatsData().getLongStats(); mColStats.setLongStats(longStats.getNumNulls(), longStats.getNumDVs(), longStats.getLowValue(), longStats.getHighValue()); } else if (statsObj.getStatsData().isSetDoubleStats()) { DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); mColStats.setDoubleStats(doubleStats.getNumNulls(), doubleStats.getNumDVs(), doubleStats.getLowValue(), doubleStats.getHighValue()); } else if (statsObj.getStatsData().isSetStringStats()) { StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats(); mColStats.setStringStats(stringStats.getNumNulls(), stringStats.getNumDVs(), stringStats.getMaxColLen(), stringStats.getAvgColLen()); } else if (statsObj.getStatsData().isSetBinaryStats()) { BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats(); mColStats.setBinaryStats(binaryStats.getNumNulls(), binaryStats.getMaxColLen(), binaryStats.getAvgColLen());
public static ColumnStatisticsDesc getTableColumnStatisticsDesc( MTableColumnStatistics mStatsObj) { ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); statsDesc.setIsTblLevel(true); statsDesc.setDbName(mStatsObj.getDbName()); statsDesc.setTableName(mStatsObj.getTableName()); statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed()); return statsDesc; }
private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { String dbName = mStatsObj.getDbName(); String tableName = mStatsObj.getTableName(); String colName = mStatsObj.getColName(); LOG.info("Updating table level column statistics for db=" + dbName + " tableName=" + tableName + " colName=" + colName); validateTableCols(table, Lists.newArrayList(colName)); List<MTableColumnStatistics> oldStats = getMTableColumnStatistics(table, Lists.newArrayList(colName)); if (!oldStats.isEmpty()) { assert oldStats.size() == 1; StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); } else { pm.makePersistent(mStatsObj); } }
private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStats) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { String colName = mStatsObj.getColName(); QueryWrapper queryWrapper = new QueryWrapper(); try { LOG.info("Updating table level column statistics for table={}" + " colName={}", getCatalogQualifiedTableName(table), colName); validateTableCols(table, Lists.newArrayList(colName)); if (oldStats != null) { StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector() == null) { // workaround for DN bug in persisting nulls in pg bytea column // instead set empty bit vector with header. mStatsObj.setBitVector(new byte[] {'H','L'}); } pm.makePersistent(mStatsObj); } } finally { queryWrapper.close(); } }
@Override protected ColumnStatistics getJdoResult( GetHelper<ColumnStatistics> ctx) throws MetaException, NoSuchObjectException { List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames); if (mStats.isEmpty()) return null; // LastAnalyzed is stored per column, but thrift object has it per multiple columns. // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now. ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0)); List<ColumnStatisticsObj> statObjs = new ArrayList<ColumnStatisticsObj>(mStats.size()); for (MTableColumnStatistics mStat : mStats) { if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) { desc.setLastAnalyzed(mStat.getLastAnalyzed()); } statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat)); Deadline.checkTimeout(); } return new ColumnStatistics(desc, statObjs); } }.run(true);
/** * Get table's column stats * * @param table * @param colNames * @return Map of column name and its stats * @throws NoSuchObjectException * @throws MetaException */ private Map<String, MTableColumnStatistics> getPartitionColStats(Table table, List<String> colNames) throws NoSuchObjectException, MetaException { Map<String, MTableColumnStatistics> statsMap = Maps.newHashMap(); QueryWrapper queryWrapper = new QueryWrapper(); try { List<MTableColumnStatistics> stats = getMTableColumnStatistics(table, colNames, queryWrapper); for(MTableColumnStatistics cStat : stats) { statsMap.put(cStat.getColName(), cStat); } } finally { queryWrapper.close(); } return statsMap; }
public static void setFieldsIntoOldStats( MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStatsObj) { if (mStatsObj.getAvgColLen() != null) { oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); if (mStatsObj.getLongHighValue() != null) { oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); if (mStatsObj.getLongLowValue() != null) { oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue()); if (mStatsObj.getDoubleLowValue() != null) { oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue()); if (mStatsObj.getDoubleHighValue() != null) { oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue()); if (mStatsObj.getDecimalLowValue() != null) { oldStatsObj.setDecimalLowValue(mStatsObj.getDecimalLowValue()); if (mStatsObj.getDecimalHighValue() != null) { oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue()); if (mStatsObj.getMaxColLen() != null) { oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); if (mStatsObj.getNumDVs() != null) { oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); if (mStatsObj.getNumFalses() != null) {
private ColumnStatisticsObj getTableColumnStatisticsObj(MTableColumnStatistics mStatsObj) { ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); statsObj.setColType(mStatsObj.getColType()); statsObj.setColName(mStatsObj.getColName()); String colType = mStatsObj.getColType(); ColumnStatisticsData colStatsData = new ColumnStatisticsData(); boolStats.setNumFalses(mStatsObj.getNumFalses()); boolStats.setNumTrues(mStatsObj.getNumTrues()); boolStats.setNumNulls(mStatsObj.getNumNulls()); colStatsData.setBooleanStats(boolStats); } else if (colType.equalsIgnoreCase("string")) { StringColumnStatsData stringStats = new StringColumnStatsData(); stringStats.setNumNulls(mStatsObj.getNumNulls()); stringStats.setAvgColLen(mStatsObj.getAvgColLen()); stringStats.setMaxColLen(mStatsObj.getMaxColLen()); stringStats.setNumDVs(mStatsObj.getNumDVs()); colStatsData.setStringStats(stringStats); } else if (colType.equalsIgnoreCase("binary")) { BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); binaryStats.setNumNulls(mStatsObj.getNumNulls()); binaryStats.setAvgColLen(mStatsObj.getAvgColLen()); binaryStats.setMaxColLen(mStatsObj.getMaxColLen()); colStatsData.setBinaryStats(binaryStats); } else if (colType.equalsIgnoreCase("bigint") || colType.equalsIgnoreCase("int") || colType.equalsIgnoreCase("timestamp")) { LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(mStatsObj.getNumNulls()); longStats.setHighValue(mStatsObj.getLongHighValue());
MTableColumnStatistics mColStats = new MTableColumnStatistics(); mColStats.setTable(table); mColStats.setDbName(statsDesc.getDbName()); mColStats.setTableName(statsDesc.getTableName()); mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); mColStats.setColName(statsObj.getColName()); mColStats.setColType(statsObj.getColType()); mColStats.setBooleanStats( boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, } else if (statsObj.getStatsData().isSetLongStats()) { LongColumnStatsData longStats = statsObj.getStatsData().getLongStats(); mColStats.setLongStats( longStats.isSetNumNulls() ? longStats.getNumNulls() : null, longStats.isSetNumDVs() ? longStats.getNumDVs() : null, } else if (statsObj.getStatsData().isSetDoubleStats()) { DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); mColStats.setDoubleStats( doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, String low = decimalStats.isSetLowValue() ? createJdoDecimalString(decimalStats.getLowValue()) : null; String high = decimalStats.isSetHighValue() ? createJdoDecimalString(decimalStats.getHighValue()) : null; mColStats.setDecimalStats( decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
private ColumnStatisticsDesc getTableColumnStatisticsDesc(MTableColumnStatistics mStatsObj) { ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); statsDesc.setIsTblLevel(true); statsDesc.setDbName(mStatsObj.getDbName()); statsDesc.setTableName(mStatsObj.getTableName()); statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed()); return statsDesc; }