protected ValidWriteIdList getMmValidWriteIds( JobConf conf, TableDesc table, ValidWriteIdList validWriteIdList) throws IOException { if (!AcidUtils.isInsertOnlyTable(table.getProperties())) return null; if (validWriteIdList == null) { validWriteIdList = AcidUtils.getTableValidWriteIdList( conf, table.getTableName()); if (validWriteIdList == null) { throw new IOException("Insert-Only table: " + table.getTableName() + " is missing from the ValidWriteIdList config: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); } } return validWriteIdList; }
String fullTableName = getFullTableName(dbName, tblName); if (sessionTxnMgr != null && sessionTxnMgr.getCurrentTxnId() > 0) { validWriteIdList = getTableValidWriteIdList(conf, fullTableName); if (isStatsUpdater) { writeId = SessionState.get().getTxnMgr() != null ?
private void copyMmPath() throws LoginException, IOException { ValidWriteIdList ids = AcidUtils.getTableValidWriteIdList(hiveConf, mmCtx.getFqTableName()); for (Path fromPath : dataPathList) { fromPath = dataFileSystem.makeQualified(fromPath);
/** * Set the valid write id list for the current table scan. */ public static void setValidWriteIdList(Configuration conf, TableScanDesc tsDesc) { if (tsDesc.isTranscationalTable()) { String dbName = tsDesc.getDatabaseName(); String tableName = tsDesc.getTableName(); ValidWriteIdList validWriteIdList = getTableValidWriteIdList(conf, AcidUtils.getFullTableName(dbName, tableName)); if (validWriteIdList != null) { setValidWriteIdList(conf, validWriteIdList); } else { // Log error if the acid table is missing from the ValidWriteIdList conf LOG.error("setValidWriteIdList on table: " + AcidUtils.getFullTableName(dbName, tableName) + " isAcidTable: " + true + " acidProperty: " + getAcidOperationalProperties(conf) + " couldn't find the ValidWriteId list from ValidTxnWriteIdList: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); throw new IllegalStateException("ACID table: " + AcidUtils.getFullTableName(dbName, tableName) + " is missing from the ValidWriteIdList config: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); } } }
public static List<FileStatus> getAcidFilesForStats( Table table, Path dir, Configuration jc, FileSystem fs) throws IOException { List<FileStatus> fileList = new ArrayList<>(); ValidWriteIdList idList = AcidUtils.getTableValidWriteIdList(jc, AcidUtils.getFullTableName(table.getDbName(), table.getTableName())); if (idList == null) {
throws IOException { String tableName = table.getTableName(); ValidWriteIdList validWriteIdList = AcidUtils.getTableValidWriteIdList( conf, tableName == null ? null : HiveStringUtils.normalizeIdentifier(tableName)); ValidWriteIdList validMmWriteIdList = getMmValidWriteIds(conf, table, validWriteIdList);
public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaException, IOException { // Construct a column statistics object from the result List<ColumnStatistics> colStats = constructColumnStatsFromPackedRows(tbl); // Persist the column statistics object to the metastore // Note, this function is shared for both table and partition column stats. if (colStats.isEmpty()) { return 0; } SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); request.setNeedMerge(colStatDesc.isNeedMerge()); HiveTxnManager txnMgr = AcidUtils.isTransactionalTable(tbl) ? SessionState.get().getTxnMgr() : null; if (txnMgr != null) { request.setValidWriteIdList(AcidUtils.getTableValidWriteIdList(conf, AcidUtils.getFullTableName(tbl.getDbName(), tbl.getTableName())).toString()); request.setWriteId(txnMgr.getAllocatedTableWriteId(tbl.getDbName(), tbl.getTableName())); } db.setPartitionColumnStatistics(request); return 0; }