public JSONUpdateTableColumnStatMessage(String server, String servicePrincipal, Long timestamp, ColumnStatistics colStats, Table tableObj, Map<String, String> parameters, String validWriteIds, long writeId) { this.timestamp = timestamp; this.server = server; this.servicePrincipal = servicePrincipal; this.writeId = writeId; this.validWriteIds = validWriteIds; this.database = colStats.getStatsDesc().getDbName(); try { this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); this.tableObjJson = MessageBuilder.createTableObjJson(tableObj); } catch (TException e) { throw new IllegalArgumentException("Could not serialize JSONUpdateTableColumnStatMessage : ", e); } this.parameters = parameters; }
public JSONUpdatePartitionColumnStatMessage(String server, String servicePrincipal, Long timestamp, ColumnStatistics colStats, List<String> partVals, Map<String, String> parameters, Table tableObj, String validWriteIds, long writeId) { this.timestamp = timestamp; this.server = server; this.servicePrincipal = servicePrincipal; this.writeId = writeId; this.validWriteIds = validWriteIds; this.database = colStats.getStatsDesc().getDbName(); this.partVals = partVals; try { this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); this.tableObjJson = MessageBuilder.createTableObjJson(tableObj); } catch (TException e) { throw new IllegalArgumentException("Could not serialize JSONUpdatePartitionColumnStatMessage : ", e); } this.parameters = parameters; }
/** {@inheritDoc} */ @Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws NoSuchObjectException, InvalidObjectException, MetaException, TException, InvalidInputException { if (request.getColStatsSize() == 1) { ColumnStatistics colStats = request.getColStatsIterator().next(); ColumnStatisticsDesc desc = colStats.getStatsDesc(); String dbName = desc.getDbName().toLowerCase(); String tableName = desc.getTableName().toLowerCase(); if (getTempTable(dbName, tableName) != null) { return updateTempTableColumnStats(dbName, tableName, colStats); } } return super.setPartitionColumnStatistics(request); }
public Object getFieldValue(_Fields field) { switch (field) { case IS_TBL_LEVEL: return isIsTblLevel(); case DB_NAME: return getDbName(); case TABLE_NAME: return getTableName(); case PART_NAME: return getPartName(); case LAST_ANALYZED: return getLastAnalyzed(); case CAT_NAME: return getCatName(); } throw new IllegalStateException(); }
/** {@inheritDoc} */ @Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws NoSuchObjectException, InvalidObjectException, MetaException, TException, InvalidInputException { if (request.getColStatsSize() == 1) { ColumnStatistics colStats = request.getColStatsIterator().next(); ColumnStatisticsDesc desc = colStats.getStatsDesc(); String dbName = desc.getDbName().toLowerCase(); String tableName = desc.getTableName().toLowerCase(); if (getTempTable(dbName, tableName) != null) { return updateTempTableColumnStats(dbName, tableName, colStats); } } return super.setPartitionColumnStatistics(request); }
private int persistColumnStats(Hive db) throws HiveException, MetaException, IOException { ColumnStatistics colStats = constructColumnStatsFromInput(); ColumnStatisticsDesc colStatsDesc = colStats.getStatsDesc(); // We do not support stats replication for a transactional table yet. If we are converting // a non-transactional table to a transactional table during replication, we might get // column statistics but we shouldn't update those. if (work.getColStats() != null && AcidUtils.isTransactionalTable(getHive().getTable(colStatsDesc.getDbName(), colStatsDesc.getTableName()))) { LOG.debug("Skipped updating column stats for table " + TableName.getDbTable(colStatsDesc.getDbName(), colStatsDesc.getTableName()) + " because it is converted to a transactional table during replication."); return 0; } SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(Collections.singletonList(colStats)); db.setPartitionColumnStatistics(request); return 0; }
private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException { // TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless. ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf)); statsDesc.setDbName(statsDesc.getDbName().toLowerCase()); statsDesc.setTableName(statsDesc.getTableName().toLowerCase()); statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName())); long time = System.currentTimeMillis() / 1000; statsDesc.setLastAnalyzed(time); for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) { statsObj.setColName(statsObj.getColName().toLowerCase()); statsObj.setColType(statsObj.getColType().toLowerCase()); } colStats.setStatsDesc(statsDesc); colStats.setStatsObj(colStats.getStatsObj()); }
public boolean setPartitionColumnStatistics( SetPartitionsStatsRequest request) throws HiveException { try { ColumnStatistics colStat = request.getColStats().get(0); ColumnStatisticsDesc statsDesc = colStat.getStatsDesc(); Table tbl = getTable(statsDesc.getDbName(), statsDesc.getTableName()); AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, true); request.setValidWriteIdList(tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); request.setWriteId(tableSnapshot != null ? tableSnapshot.getWriteId() : 0); return getMSC().setPartitionColumnStatistics(request); } catch (Exception e) { LOG.debug(StringUtils.stringifyException(e)); throw new HiveException(e); } }
ColumnStatistics colStats = work.getColStats(); LOG.debug("Got stats through replication for " + colStats.getStatsDesc().getDbName() + "." + colStats.getStatsDesc().getTableName()); return colStats;
ColumnStatisticsDesc statsDesc = firstColStats.getStatsDesc(); String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); String dbName = statsDesc.getDbName(); String tableName = statsDesc.getTableName(); List<String> colNames = new ArrayList<>();
ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); Table table = ensureGetTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); Partition partition = convertToPart(getMPartition( catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals)); List<String> colNames = new ArrayList<>(); catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals); if (partition == null) { throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); + statsDesc.getDbName() + "." + statsDesc.getTableName() + "." + statsDesc.getPartName());
Table table = ensureGetTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); List<String> colNames = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj); writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName()));
colStats.getStatsDesc().getCatName(), colStats.getStatsDesc().getDbName(), colStats.getStatsDesc().getTableName())); if (parameters != null) { Table tableObj = getMS().getTable(colStats.getStatsDesc().getCatName(), colStats.getStatsDesc().getDbName(), colStats.getStatsDesc().getTableName(), validWriteIds); if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
String catName = csd.getCatName(), dbName = csd.getDbName(), tableName = csd.getTableName(); startFunction("write_partition_column_statistics", ": db=" + dbName + " table=" + tableName + " part=" + csd.getPartName());
@Override public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats, List<String> partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { Map<String, String> newParams = rawStore.updatePartitionColumnStatistics( colStats, partVals, validWriteIds, writeId); // in case of event based cache update, cache is updated during commit txn if (newParams != null && !canUseEvents) { String catName = colStats.getStatsDesc().isSetCatName() ? normalizeIdentifier(colStats.getStatsDesc().getCatName()) : DEFAULT_CATALOG_NAME; String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName()); if (!shouldCacheTable(catName, dbName, tblName)) { return newParams; } Partition part = getPartition(catName, dbName, tblName, partVals); part.setParameters(newParams); sharedCache.alterPartitionInCache(catName, dbName, tblName, partVals, part); sharedCache.updatePartitionColStatsInCache(catName, dbName, tblName, partVals, colStats.getStatsObj()); } return newParams; }
@Override public void onUpdateTableColumnStat(UpdateTableColumnStatEvent updateTableColumnStatEvent) throws MetaException { UpdateTableColumnStatMessage msg = MessageBuilder.getInstance() .buildUpdateTableColumnStatMessage(updateTableColumnStatEvent.getColStats(), updateTableColumnStatEvent.getTableObj(), updateTableColumnStatEvent.getTableParameters(), updateTableColumnStatEvent.getValidWriteIds(), updateTableColumnStatEvent.getWriteId()); NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_TABLE_COLUMN_STAT.toString(), msgEncoder.getSerializer().serialize(msg)); ColumnStatisticsDesc statDesc = updateTableColumnStatEvent.getColStats().getStatsDesc(); event.setCatName(statDesc.isSetCatName() ? statDesc.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(statDesc.getDbName()); event.setTableName(statDesc.getTableName()); process(event, updateTableColumnStatEvent); }
@Override public void onUpdatePartitionColumnStat(UpdatePartitionColumnStatEvent updatePartColStatEvent) throws MetaException { UpdatePartitionColumnStatMessage msg = MessageBuilder.getInstance() .buildUpdatePartitionColumnStatMessage(updatePartColStatEvent.getPartColStats(), updatePartColStatEvent.getPartVals(), updatePartColStatEvent.getPartParameters(), updatePartColStatEvent.getTableObj(), updatePartColStatEvent.getValidWriteIds(), updatePartColStatEvent.getWriteId()); NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_PARTITION_COLUMN_STAT.toString(), msgEncoder.getSerializer().serialize(msg)); ColumnStatisticsDesc statDesc = updatePartColStatEvent.getPartColStats().getStatsDesc(); event.setCatName(statDesc.isSetCatName() ? statDesc.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(statDesc.getDbName()); event.setTableName(statDesc.getTableName()); process(event, updatePartColStatEvent); }
normalizeIdentifier(colStats.getStatsDesc().getCatName()) : getDefaultCatalog(conf); String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName()); if (!shouldCacheTable(catName, dbName, tblName)) {
mColStats.setPartition(partition); mColStats.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName() : DEFAULT_CATALOG_NAME); mColStats.setDbName(statsDesc.getDbName()); mColStats.setTableName(statsDesc.getTableName()); mColStats.setPartitionName(statsDesc.getPartName());
mColStats.setDbName(statsDesc.getDbName()); mColStats.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName() : DEFAULT_CATALOG_NAME); mColStats.setTableName(statsDesc.getTableName());