/** * Convertes to DateColumnStatsDataInspector if it's a DateColumnStatsData. * @param cso ColumnStatisticsObj * @return DateColumnStatsDataInspector */ public static DateColumnStatsDataInspector dateInspectorFromStats(ColumnStatisticsObj cso) { DateColumnStatsDataInspector dateColumnStats; if (cso.getStatsData().getDateStats() instanceof DateColumnStatsDataInspector) { dateColumnStats = (DateColumnStatsDataInspector)(cso.getStatsData().getDateStats()); } else { dateColumnStats = new DateColumnStatsDataInspector(cso.getStatsData().getDateStats()); } return dateColumnStats; }
private static void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(buf); ; } }
@Override void compare(ColumnStatisticsData colstats, int offset) { Assert.assertEquals("date num nulls", numNulls.get(offset), (Long) colstats.getDateStats().getNumNulls()); Assert.assertEquals("date num dvs", numDvs.get(offset), (Long) colstats.getDateStats().getNumDVs()); Assert.assertEquals("date low val", lowVals.get(offset), colstats.getDateStats().getLowValue()); Assert.assertEquals("date high val", highVals.get(offset), colstats.getDateStats().getHighValue()); }
private void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(v);; } }
Map<String, DateColumnStatsData> extractedAdjustedStatsMap = new HashMap<>(); for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) { extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDateStats());
@Override void compareAggr(ColumnStatisticsObj obj) { compareCommon(obj); Assert.assertEquals("aggr date num nulls", getNumNulls(), obj.getStatsData().getDateStats().getNumNulls()); Assert.assertEquals("aggr date num dvs", getNumDvs(), obj.getStatsData().getDateStats().getNumDVs()); Assert.assertEquals("aggr date low val", getLowVal(), obj.getStatsData().getDateStats().getLowValue()); Assert.assertEquals("aggr date high val", getHighVal(), obj.getStatsData().getDateStats().getHighValue()); }
@Test public void testMergeNullMinMaxValues() { ColumnStatisticsObj old = new ColumnStatisticsObj(); createData(old, null, null); merger.merge(old, old); Assert.assertNull(old.getStatsData().getDateStats().getLowValue()); Assert.assertNull(old.getStatsData().getDateStats().getHighValue()); }
convertToString(lcsd.getBitVectors()))); } else if (csd.isSetDateStats()) { DateColumnStatsData dcsd = csd.getDateStats(); ret.addAll(Lists.newArrayList( convertToString(dcsd.getLowValue()), convertToString(dcsd.getHighValue()), "" + dcsd.getNumNulls(), "" + dcsd.getNumDVs(),
ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); DateColumnStatsData newData = cso.getStatsData().getDateStats(); if (useDensityFunctionForNDVEstimation) { densityAvgSum += diff(newData.getHighValue(), newData.getLowValue()) / newData.getNumDVs(); colName, columnStatisticsData.getDateStats().getNumDVs(), partNames.size(), colStatsWithSourceInfo.size()); statsObj.setStatsData(columnStatisticsData);
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integer : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); case Date: return statData.getDateStats().getNumNulls(); default: return null; } }
@Test public void testMergeNulls() { ColumnStatisticsObj oldObj = new ColumnStatisticsObj(); createData(oldObj, null, null); ColumnStatisticsObj newObj; newObj = new ColumnStatisticsObj(); createData(newObj, null, null); merger.merge(oldObj, newObj); Assert.assertEquals(null, oldObj.getStatsData().getDateStats().getLowValue()); Assert.assertEquals(null, oldObj.getStatsData().getDateStats().getHighValue()); newObj = new ColumnStatisticsObj(); createData(newObj, DATE_1, DATE_3); merger.merge(oldObj, newObj); newObj = new ColumnStatisticsObj(); createData(newObj, null, null); merger.merge(oldObj, newObj); Assert.assertEquals(DATE_1, oldObj.getStatsData().getDateStats().getLowValue()); Assert.assertEquals(DATE_3, oldObj.getStatsData().getDateStats().getHighValue()); }
@Test public void testMergeNonNullAndNullLowerValuesNewIsNull() { ColumnStatisticsObj oldObj = new ColumnStatisticsObj(); createData(oldObj, DATE_2, DATE_2); ColumnStatisticsObj newObj; newObj = new ColumnStatisticsObj(); createData(newObj, DATE_3, DATE_3); merger.merge(oldObj, newObj); newObj = new ColumnStatisticsObj(); createData(newObj, DATE_1, DATE_1); merger.merge(oldObj, newObj); Assert.assertEquals(DATE_1, oldObj.getStatsData().getDateStats().getLowValue()); Assert.assertEquals(DATE_3, oldObj.getStatsData().getDateStats().getHighValue()); }
lcsd.getNumDVs(), "", "", "", ""); } else if (csd.isSetDateStats()) { DateColumnStatsData dcsd = csd.getDateStats(); appendColumnStats(tableInfo, convertToString(dcsd.getLowValue()),
cs.setNumNulls(csd.getDateStats().getNumNulls()); Long lowVal = (csd.getDateStats().getLowValue() != null) ? csd.getDateStats().getLowValue() .getDaysSinceEpoch() : null; Long highVal = (csd.getDateStats().getHighValue() != null) ? csd.getDateStats().getHighValue() .getDaysSinceEpoch() : null; cs.setRange(lowVal, highVal);
cs.setNumNulls(csd.getDateStats().getNumNulls()); Long lowVal = (csd.getDateStats().getLowValue() != null) ? csd.getDateStats().getLowValue() .getDaysSinceEpoch() : null; Long highVal = (csd.getDateStats().getHighValue() != null) ? csd.getDateStats().getHighValue() .getDaysSinceEpoch() : null; cs.setRange(lowVal, highVal);
DateColumnStatsData dateStatsData = columnStatistics.getStatsData().getDateStats(); Optional<LocalDate> min = dateStatsData.isSetLowValue() ? fromMetastoreDate(dateStatsData.getLowValue()) : Optional.empty(); Optional<LocalDate> max = dateStatsData.isSetHighValue() ? fromMetastoreDate(dateStatsData.getHighValue()) : Optional.empty();
lcsd.getNumNulls(), lcsd.getNumDVs(), "", "", "", ""); } else if (csd.isSetDateStats()) { DateColumnStatsData dcsd = csd.getDateStats(); appendColumnStatsNoFormatting(colBuffer, convertToString(dcsd.getLowValue()),
DateColumnStatsData oldDateStatsData = oldStatObj.getStatsData().getDateStats(); DateColumnStatsData newDateStatsData = newStatObj.getStatsData().getDateStats(); if (newDateStatsData.isSetHighValue()) { oldDateStatsData.setHighValue(newDateStatsData.getHighValue());
binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null); } else if (statsObj.getStatsData().isSetDateStats()) { DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats(); mColStats.setDateStats( dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null,
binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null); } else if (statsObj.getStatsData().isSetDateStats()) { DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats(); mColStats.setDateStats( dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null,