@Override public double getRowCount() { if (rowCount == -1) { if (null == partitionList) { // we are here either unpartitioned table or partitioned table with no // predicates computePartitionList(hiveConf, null, new HashSet<Integer>()); } if (hiveTblMetadata.isPartitioned()) { List<Long> rowCounts = StatsUtils.getBasicStatForPartitions(hiveTblMetadata, partitionList.getNotDeniedPartns(), StatsSetupConst.ROW_COUNT); rowCount = StatsUtils.getSumIgnoreNegatives(rowCounts); } else { rowCount = StatsUtils.getNumRows(hiveTblMetadata); } } if (rowCount == -1) noColsMissingStats.getAndIncrement(); return rowCount; }
private Status checkThresholdWithMetastoreStats(final Table table, final PrunedPartitionList partsList, final long threshold) { if (table != null && !table.isPartitioned()) { long dataSize = StatsUtils.getTotalSize(table); if (dataSize <= 0) { LOG.warn("Cannot determine basic stats for table: {} from metastore. Falling back.", table.getCompleteName()); return Status.UNAVAILABLE; } return (threshold - dataSize) >= 0 ? Status.PASS : Status.FAIL; } else if (table != null && table.isPartitioned() && partsList != null) { List<Long> dataSizes = StatsUtils.getBasicStatForPartitions(table, partsList.getNotDeniedPartns(), StatsSetupConst.TOTAL_SIZE); long totalDataSize = StatsUtils.getSumIgnoreNegatives(dataSizes); if (totalDataSize <= 0) { LOG.warn("Cannot determine basic stats for partitioned table: {} from metastore. Falling back.", table.getCompleteName()); return Status.UNAVAILABLE; } return (threshold - totalDataSize) >= 0 ? Status.PASS : Status.FAIL; } return Status.UNAVAILABLE; }
private Status checkThresholdWithMetastoreStats(final Table table, final PrunedPartitionList partsList, final long threshold) { Status status = Status.UNAVAILABLE; if (table != null && !table.isPartitioned()) { long dataSize = StatsUtils.getTotalSize(table); if (dataSize <= 0) { LOG.warn("Cannot determine basic stats for table: {} from metastore. Falling back.", table.getCompleteName()); return Status.UNAVAILABLE; } status = (threshold - dataSize) >= 0 ? Status.PASS : Status.FAIL; } else if (table != null && table.isPartitioned() && partsList != null) { List<Long> dataSizes = StatsUtils.getBasicStatForPartitions(table, partsList.getNotDeniedPartns(), StatsSetupConst.TOTAL_SIZE); long totalDataSize = StatsUtils.getSumIgnoreNegatives(dataSizes); if (totalDataSize <= 0) { LOG.warn("Cannot determine basic stats for partitioned table: {} from metastore. Falling back.", table.getCompleteName()); return Status.UNAVAILABLE; } status = (threshold - totalDataSize) >= 0 ? Status.PASS : Status.FAIL; } if (status == Status.PASS && MetaStoreUtils.isExternalTable(table.getTTable())) { // External table should also check the underlying file size. LOG.warn("Table {} is external table, falling back to filesystem scan.", table.getCompleteName()); status = Status.UNAVAILABLE; } return status; }
long ds = bbs.getDataSize(); List<Long> erasureCodedFiles = getBasicStatForPartitions(table, partList.getNotDeniedPartns(), StatsSetupConst.NUM_ERASURE_CODED_FILES); long numErasureCodedFiles = getSumIgnoreNegatives(erasureCodedFiles);
rowCounts = getBasicStatForPartitions( table, partList.getNotDeniedPartns(), StatsSetupConst.ROW_COUNT); dataSizes = getBasicStatForPartitions( table, partList.getNotDeniedPartns(), StatsSetupConst.RAW_DATA_SIZE); ds = getSumIgnoreNegatives(dataSizes); if (ds <= 0) { dataSizes = getBasicStatForPartitions( table, partList.getNotDeniedPartns(), StatsSetupConst.TOTAL_SIZE); ds = getSumIgnoreNegatives(dataSizes);
@Override public double getRowCount() { if (rowCount == -1) { if (null == partitionList) { // we are here either unpartitioned table or partitioned table with no // predicates computePartitionList(hiveConf, null); } if (hiveTblMetadata.isPartitioned()) { List<Long> rowCounts = StatsUtils.getBasicStatForPartitions(hiveTblMetadata, partitionList.getNotDeniedPartns(), StatsSetupConst.ROW_COUNT); rowCount = StatsUtils.getSumIgnoreNegatives(rowCounts); } else { rowCount = StatsUtils.getNumRows(hiveTblMetadata); } } if (rowCount == -1) noColsMissingStats.getAndIncrement(); return rowCount; }
rowCounts = getBasicStatForPartitions( table, partList.getNotDeniedPartns(), StatsSetupConst.ROW_COUNT); dataSizes = getBasicStatForPartitions( table, partList.getNotDeniedPartns(), StatsSetupConst.RAW_DATA_SIZE); ds = getSumIgnoreNegatives(dataSizes); if (ds <= 0) { dataSizes = getBasicStatForPartitions( table, partList.getNotDeniedPartns(), StatsSetupConst.TOTAL_SIZE); ds = getSumIgnoreNegatives(dataSizes);