private List<String> getExistingNonPartTableStatsToUpdate(TableName fullTableName, String cat, String db, String tbl, Map<String, String> params, long statsWriteId, List<String> allCols, String writeIdString) throws MetaException { ColumnStatistics existingStats = null; try { // Note: this should NOT do txn verification - we want to get outdated stats, to // see if we need to update anything. existingStats = rs.getTableColumnStatistics(cat, db, tbl, allCols); } catch (NoSuchObjectException e) { LOG.error("Cannot retrieve existing stats, skipping " + fullTableName, e); return null; } // TODO: we should probably skip updating if writeId is from an active txn boolean isTxnValid = (writeIdString == null) || ObjectStore.isCurrentStatsValidForTheQuery( conf, params, statsWriteId , writeIdString, false); return getExistingStatsToUpdate(existingStats, params, isTxnValid); }
@Deprecated @Override public ColumnStatistics get_table_column_statistics(String dbName, String tableName, String colName) throws TException { String[] parsedDbName = parseDbName(dbName, conf); parsedDbName[CAT_NAME] = parsedDbName[CAT_NAME].toLowerCase(); parsedDbName[DB_NAME] = parsedDbName[DB_NAME].toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); startFunction("get_column_statistics_by_table", ": table=" + TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) + " column=" + colName); ColumnStatistics statsObj = null; try { statsObj = getMS().getTableColumnStatistics( parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(colName), null); if (statsObj != null) { assert statsObj.getStatsObjSize() <= 1; } return statsObj; } finally { endFunction("get_column_statistics_by_table", statsObj != null, null, tableName); } }
@Override public TableStatsResult get_table_statistics_req(TableStatsRequest request) throws TException { String catName = request.isSetCatName() ? request.getCatName().toLowerCase() : getDefaultCatalog(conf); String dbName = request.getDbName().toLowerCase(); String tblName = request.getTblName().toLowerCase(); startFunction("get_table_statistics_req", ": table=" + TableName.getQualified(catName, dbName, tblName)); TableStatsResult result = null; List<String> lowerCaseColNames = new ArrayList<>(request.getColNames().size()); for (String colName : request.getColNames()) { lowerCaseColNames.add(colName.toLowerCase()); } try { ColumnStatistics cs = getMS().getTableColumnStatistics( catName, dbName, tblName, lowerCaseColNames, request.getValidWriteIdList()); // Note: stats compliance is not propagated to the client; instead, we just return nothing // if stats are not compliant for now. This won't work for stats merging, but that // is currently only done on metastore size (see set_aggr...). // For some optimizations we might make use of incorrect stats that are "better than // nothing", so this may change in future. result = new TableStatsResult((cs == null || cs.getStatsObj() == null || (cs.isSetIsStatsCompliant() && !cs.isIsStatsCompliant())) ? Lists.newArrayList() : cs.getStatsObj()); } finally { endFunction("get_table_statistics_req", result == null, null, tblName); } return result; }
@Override public ColumnStatistics getTableColumnStatistics( String catName, String dbName, String tblName, List<String> colNames, String validWriteIds) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { return rawStore.getTableColumnStatistics( catName, dbName, tblName, colNames, validWriteIds); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache return rawStore.getTableColumnStatistics( catName, dbName, tblName, colNames, validWriteIds); } ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tblName); List<ColumnStatisticsObj> colStatObjs = sharedCache.getTableColStatsFromCache(catName, dbName, tblName, colNames); return adjustColStatForGet(table.getParameters(), table.getParameters(), new ColumnStatistics(csd, colStatObjs), table.getWriteId(), validWriteIds); }
Deadline.startTimer("getTableColumnStatistics"); tableColStats = rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer();
colStats = msdb.getTableColumnStatistics(catName, dbName, tableName, oldColNames); if (colStats == null) { updateColumnStats = false;
public Table get_table_core(final String catName, final String dbname, final String name, final String writeIdList, boolean getColumnStats) throws MetaException, NoSuchObjectException { Table t = null; try { t = getMS().getTable(catName, dbname, name, writeIdList); if (t == null) { throw new NoSuchObjectException(TableName.getQualified(catName, dbname, name) + " table not found"); } // If column statistics was requested and is valid fetch it. if (getColumnStats) { ColumnStatistics colStats = getMS().getTableColumnStatistics(catName, dbname, name, StatsSetupConst.getColumnsHavingStats(t.getParameters()), writeIdList); if (colStats != null) { t.setColStats(colStats); } } } catch (Exception e) { throwMetaException(e); } return t; }
boolean isCommitted = false, result = false; try { ColumnStatistics csOld = ms.getTableColumnStatistics(catName, dbName, tableName, colNames, request.getValidWriteIdList());
@Test public void testAlterTableAddColNotUpdateStats() throws MetaException, InvalidObjectException, NoSuchObjectException { FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); FieldSchema col4 = new FieldSchema("col4", "string", "col4 comment"); StorageDescriptor oldSd = new StorageDescriptor(); oldSd.setCols(Arrays.asList(col1, col2, col3)); Table oldTable = new Table(); oldTable.setDbName("default"); oldTable.setTableName("test_table"); oldTable.setSd(oldSd); StorageDescriptor newSd = new StorageDescriptor(oldSd); newSd.setCols(Arrays.asList(col1, col2, col3, col4)); Table newTable = new Table(oldTable); newTable.setSd(newSd); RawStore msdb = Mockito.mock(RawStore.class); Mockito.doThrow(new RuntimeException("shouldn't be called")).when(msdb).getTableColumnStatistics( getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); }
private void updateTableColStats(RawStore rawStore, String catName, String dbName, String tblName) { boolean committed = false; rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); if (!table.isSetPartitionKeys()) { List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); ColumnStatistics tableColStats = rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); if (tableColStats != null) { sharedCache.refreshTableColStatsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), tableColStats.getStatsObj()); // Update the table to get consistent stats state. sharedCache.alterTableInCache(catName, dbName, tblName, table); } } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Unable to refresh table column stats for table: " + tblName, e); } finally { if (!committed) { sharedCache.removeAllTableColStatsFromCache(catName, dbName, tblName); rawStore.rollbackTransaction(); } } }
@Test public void testAlterTableChangePosNotUpdateStats() throws MetaException, InvalidObjectException, NoSuchObjectException { FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); FieldSchema col4 = new FieldSchema("col4", "string", "col4 comment"); StorageDescriptor oldSd = new StorageDescriptor(); oldSd.setCols(Arrays.asList(col1, col2, col3, col4)); Table oldTable = new Table(); oldTable.setDbName("default"); oldTable.setTableName("test_table"); oldTable.setSd(oldSd); StorageDescriptor newSd = new StorageDescriptor(oldSd); newSd.setCols(Arrays.asList(col1, col4, col2, col3)); Table newTable = new Table(oldTable); newTable.setSd(newSd); RawStore msdb = Mockito.mock(RawStore.class); Mockito.doThrow(new RuntimeException("shouldn't be called")).when(msdb).getTableColumnStatistics( getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); }
@Test public void testAlterTableDelColUpdateStats() throws Exception { FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); FieldSchema col4 = new FieldSchema("col4", "string", "col4 comment"); StorageDescriptor oldSd = new StorageDescriptor(); oldSd.setCols(Arrays.asList(col1, col2, col3, col4)); Table oldTable = new Table(); oldTable.setDbName("default"); oldTable.setTableName("test_table"); oldTable.setSd(oldSd); StorageDescriptor newSd = new StorageDescriptor(oldSd); newSd.setCols(Arrays.asList(col1, col2, col3)); Table newTable = new Table(oldTable); newTable.setSd(newSd); RawStore msdb = Mockito.mock(RawStore.class); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); try { handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } catch (Throwable t) { System.err.println(t); t.printStackTrace(System.err); throw t; } Mockito.verify(msdb, Mockito.times(1)).getTableColumnStatistics( getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4") ); }
public ColumnStatistics get_table_column_statistics(String dbName, String tableName, String colName) throws NoSuchObjectException, MetaException, TException, InvalidInputException, InvalidObjectException { dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); startFunction("get_column_statistics_by_table: db=" + dbName + " table=" + tableName + " column=" + colName); ColumnStatistics statsObj = null; try { statsObj = getMS().getTableColumnStatistics(dbName, tableName, colName); } finally { endFunction("get_column_statistics_by_table: ", statsObj != null, null, tableName); } return statsObj; }
@Override public ColumnStatistics get_table_column_statistics(String dbName, String tableName, String colName) throws NoSuchObjectException, MetaException, TException, InvalidInputException, InvalidObjectException { dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); startFunction("get_column_statistics_by_table: db=" + dbName + " table=" + tableName + " column=" + colName); ColumnStatistics statsObj = null; try { statsObj = getMS().getTableColumnStatistics( dbName, tableName, Lists.newArrayList(colName)); assert statsObj.getStatsObjSize() <= 1; return statsObj; } finally { endFunction("get_column_statistics_by_table: ", statsObj != null, null, tableName); } }
@Override public ColumnStatistics get_table_column_statistics(String dbName, String tableName, String colName) throws NoSuchObjectException, MetaException, TException, InvalidInputException, InvalidObjectException { dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); startFunction("get_column_statistics_by_table: db=" + dbName + " table=" + tableName + " column=" + colName); ColumnStatistics statsObj = null; try { statsObj = getMS().getTableColumnStatistics( dbName, tableName, Lists.newArrayList(colName)); assert statsObj.getStatsObjSize() <= 1; return statsObj; } finally { endFunction("get_column_statistics_by_table: ", statsObj != null, null, tableName); } }
@Override public ColumnStatistics get_table_column_statistics(String dbName, String tableName, String colName) throws TException { String[] parsedDbName = parseDbName(dbName, conf); parsedDbName[CAT_NAME] = parsedDbName[CAT_NAME].toLowerCase(); parsedDbName[DB_NAME] = parsedDbName[DB_NAME].toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); startFunction("get_column_statistics_by_table", ": table=" + getCatalogQualifiedTableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) + " column=" + colName); ColumnStatistics statsObj = null; try { statsObj = getMS().getTableColumnStatistics( parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(colName)); if (statsObj != null) { assert statsObj.getStatsObjSize() <= 1; } return statsObj; } finally { endFunction("get_column_statistics_by_table", statsObj != null, null, tableName); } }
@Override public TableStatsResult get_table_statistics_req(TableStatsRequest request) throws MetaException, NoSuchObjectException, TException { String dbName = request.getDbName().toLowerCase(); String tblName = request.getTblName().toLowerCase(); startFunction("get_table_statistics_req: db=" + dbName + " table=" + tblName); TableStatsResult result = null; List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size()); for (String colName : request.getColNames()) { lowerCaseColNames.add(colName.toLowerCase()); } try { ColumnStatistics cs = getMS().getTableColumnStatistics(dbName, tblName, lowerCaseColNames); result = new TableStatsResult( cs == null ? Lists.<ColumnStatisticsObj>newArrayList() : cs.getStatsObj()); } finally { endFunction("get_table_statistics_req: ", result == null, null, tblName); } return result; }
@Override public TableStatsResult get_table_statistics_req(TableStatsRequest request) throws MetaException, NoSuchObjectException, TException { String dbName = request.getDbName().toLowerCase(); String tblName = request.getTblName().toLowerCase(); startFunction("get_table_statistics_req: db=" + dbName + " table=" + tblName); TableStatsResult result = null; List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size()); for (String colName : request.getColNames()) { lowerCaseColNames.add(colName.toLowerCase()); } try { ColumnStatistics cs = getMS().getTableColumnStatistics(dbName, tblName, lowerCaseColNames); result = new TableStatsResult( cs == null ? Lists.<ColumnStatisticsObj>newArrayList() : cs.getStatsObj()); } finally { endFunction("get_table_statistics_req: ", result == null, null, tblName); } return result; }
@Override public ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tblName, List<String> colNames) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { return rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache return rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); } ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tblName); List<ColumnStatisticsObj> colStatObjs = sharedCache.getTableColStatsFromCache(catName, dbName, tblName, colNames); return new ColumnStatistics(csd, colStatObjs); }
private void updateTableColStats(RawStore rawStore, String catName, String dbName, String tblName) { try { Table table = rawStore.getTable(catName, dbName, tblName); if (!table.isSetPartitionKeys()) { List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); ColumnStatistics tableColStats = rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); if (tableColStats != null) { sharedCache.refreshTableColStatsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), tableColStats.getStatsObj()); } } } catch (MetaException | NoSuchObjectException e) { LOG.info("Unable to refresh table column stats for table: " + tblName, e); } }