static ColumnStatisticsObj getColumnStatisticsObject(String colName, String colType, List<ColumnStatisticsObj> colStats) { if (colStats != null && !colStats.isEmpty()) { for (ColumnStatisticsObj cso : colStats) { if (cso.getColName().equalsIgnoreCase(colName) && cso.getColType().equalsIgnoreCase(colType)) { return cso; } } } return null; }
private static ColumnStatisticsObj getColumnStatisticsObject(String colName, String colType, List<ColumnStatisticsObj> colStats) { if (colStats != null && !colStats.isEmpty()) { for (ColumnStatisticsObj cso : colStats) { if (cso.getColName().equalsIgnoreCase(colName) && cso.getColType().equalsIgnoreCase(colType)) { return cso; } } } return null; }
public Object getFieldValue(_Fields field) { switch (field) { case COL_NAME: return getColName(); case COL_TYPE: return getColType(); case STATS_DATA: return getStatsData(); } throw new IllegalStateException(); }
private ColumnStatisticsObj prepareCSObj (Object[] row, int i) throws MetaException { ColumnStatisticsData data = new ColumnStatisticsData(); ColumnStatisticsObj cso = new ColumnStatisticsObj((String)row[i++], (String)row[i++], data); Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++], declow = row[i++], dechigh = row[i++], nulls = row[i++], dist = row[i++], bitVector = row[i++], avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i++]; StatObjectConverter.fillColumnStatisticsData(cso.getColType(), data, llow, lhigh, dlow, dhigh, declow, dechigh, nulls, dist, bitVector, avglen, maxlen, trues, falses); return cso; }
private ColumnStatisticsObj prepareCSObjWithAdjustedNDV(Object[] row, int i, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException { ColumnStatisticsData data = new ColumnStatisticsData(); ColumnStatisticsObj cso = new ColumnStatisticsObj((String) row[i++], (String) row[i++], data); Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++], declow = row[i++], dechigh = row[i++], nulls = row[i++], dist = row[i++], avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i++], avgLong = row[i++], avgDouble = row[i++], avgDecimal = row[i++], sumDist = row[i++]; StatObjectConverter.fillColumnStatisticsData(cso.getColType(), data, llow, lhigh, dlow, dhigh, declow, dechigh, nulls, dist, avglen, maxlen, trues, falses, avgLong, avgDouble, avgDecimal, sumDist, useDensityFunctionForNDVEstimation, ndvTuner); return cso; }
private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException { // TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless. ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf)); statsDesc.setDbName(statsDesc.getDbName().toLowerCase()); statsDesc.setTableName(statsDesc.getTableName().toLowerCase()); statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName())); long time = System.currentTimeMillis() / 1000; statsDesc.setLastAnalyzed(time); for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) { statsObj.setColName(statsObj.getColName().toLowerCase()); statsObj.setColType(statsObj.getColType().toLowerCase()); } colStats.setStatsDesc(statsDesc); colStats.setStatsObj(colStats.getStatsObj()); }
void compareCommon(ColumnStatisticsObj obj) { Assert.assertEquals(colName, obj.getColName()); Assert.assertEquals(colType, obj.getColType()); }
for (FieldSchema newCol : newCols) { if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) && statsObj.getColType().equalsIgnoreCase(newCol.getType())) { found = true; break;
for (FieldSchema newCol : newCols) { if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) && statsObj.getColType().equalsIgnoreCase(newCol.getType())) { found = true; break;
if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField());
if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField());
if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField());
@Override public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo, List<String> partNames, boolean areAllPartsFound) throws MetaException { ColumnStatisticsObj statsObj = null; String colType = null; String colName = null; BinaryColumnStatsData aggregateData = null; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField()); } BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats(); if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); } } ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData(); columnStatisticsData.setBinaryStats(aggregateData); statsObj.setStatsData(columnStatisticsData); return statsObj; } }
if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField());
@Override public void checkStats(AggrStats aggrStats) throws Exception { Assert.assertEquals(10, aggrStats.getPartsFound()); Assert.assertEquals(1, aggrStats.getColStatsSize()); ColumnStatisticsObj cso = aggrStats.getColStats().get(0); Assert.assertEquals("col1", cso.getColName()); Assert.assertEquals("bigint", cso.getColType()); LongColumnStatsData lcsd = cso.getStatsData().getLongStats(); Assert.assertEquals(1009, lcsd.getHighValue(), 0.01); Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01); Assert.assertEquals(45, lcsd.getNumNulls()); Assert.assertEquals(91, lcsd.getNumDVs()); } };
String colTypeLowerCase = cso.getColType().toLowerCase(); ColStatistics cs = new ColStatistics(colName, colTypeLowerCase); ColumnStatisticsData csd = cso.getStatsData();
@Override public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo, List<String> partNames, boolean areAllPartsFound) throws MetaException { ColumnStatisticsObj statsObj = null; String colType = null; String colName = null; BooleanColumnStatsData aggregateData = null; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField()); } BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats(); if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues()); aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses()); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); } } ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData(); columnStatisticsData.setBooleanStats(aggregateData); statsObj.setStatsData(columnStatisticsData); return statsObj; }
String colTypeLowerCase = cso.getColType().toLowerCase(); ColStatistics cs = new ColStatistics(colName, colTypeLowerCase); ColumnStatisticsData csd = cso.getStatsData();
mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); mColStats.setColName(statsObj.getColName()); mColStats.setColType(statsObj.getColType());
mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); mColStats.setColName(statsObj.getColName()); mColStats.setColType(statsObj.getColType());