/** * Convertes to DoubleColumnStatsDataInspector * if it's a DoubleColumnStatsData. * @param cso ColumnStatisticsObj * @return DoubleColumnStatsDataInspector */ public static DoubleColumnStatsDataInspector doubleInspectorFromStats(ColumnStatisticsObj cso) { DoubleColumnStatsDataInspector columnStats; if (cso.getStatsData().getDoubleStats() instanceof DoubleColumnStatsDataInspector) { columnStats = (DoubleColumnStatsDataInspector)(cso.getStatsData().getDoubleStats()); } else { columnStats = new DoubleColumnStatsDataInspector(cso.getStatsData().getDoubleStats()); } return columnStats; }
private static void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setLowValue(d); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(buf); ; } }
private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setLowValue(d); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(v);; } }
Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>(); for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) { extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
@Override void compare(ColumnStatisticsData colstats, int offset) { Assert.assertEquals("double num nulls", numNulls.get(offset), (Long) colstats.getDoubleStats().getNumNulls()); Assert.assertEquals("double num dvs", numDvs.get(offset), (Long) colstats.getDoubleStats().getNumDVs()); Assert.assertEquals("double low val", lowVals.get(offset), colstats.getDoubleStats().getLowValue(), 0.01); Assert.assertEquals("double high val", highVals.get(offset), colstats.getDoubleStats().getHighValue(), 0.01); }
@Override void compareAggr(ColumnStatisticsObj obj) { compareCommon(obj); Assert.assertEquals("aggr double num nulls", getNumNulls(), obj.getStatsData().getDoubleStats().getNumNulls()); Assert.assertEquals("aggr double num dvs", getNumDvs(), obj.getStatsData().getDoubleStats().getNumDVs()); Assert.assertEquals("aggr double low val", getLowVal(), obj.getStatsData().getDoubleStats().getLowValue(), 0.01); Assert.assertEquals("aggr double high val", getHighVal(), obj.getStatsData().getDoubleStats().getHighValue(), 0.01); }
ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats(); if (useDensityFunctionForNDVEstimation) { densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs(); colName, columnStatisticsData.getDoubleStats().getNumDVs(), partNames.size(), colStatsWithSourceInfo.size()); statsObj.setStatsData(columnStatisticsData);
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integeral : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); default: return null; } }
convertToString(dcsd.getBitVectors()))); } else if (csd.isSetDoubleStats()) { DoubleColumnStatsData dcsd = csd.getDoubleStats(); ret.addAll(Lists.newArrayList( "" + dcsd.getLowValue(), "" + dcsd.getHighValue(), "" + dcsd.getNumNulls(), "" + dcsd.getNumDVs(),
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integer : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); case Date: return statData.getDateStats().getNumNulls(); default: return null; } }
DoubleColumnStatsData dstats = statData.getDoubleStats(); if (dstats.isSetHighValue()) { oneRow.add(subType.cast(dstats.getHighValue())); ColumnStatisticsData statData = validateSingleColStat(statObj); if (statData == null) return null; DoubleColumnStatsData dstats = statData.getDoubleStats(); if (!dstats.isSetHighValue()) { continue; double curVal = statData.getDoubleStats().getHighValue(); maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); DoubleColumnStatsData dstats = statData.getDoubleStats(); if (dstats.isSetLowValue()) { oneRow.add(subType.cast(dstats.getLowValue())); ColumnStatisticsData statData = validateSingleColStat(statObj); if (statData == null) return null; DoubleColumnStatsData dstats = statData.getDoubleStats(); if (!dstats.isSetLowValue()) { continue; double curVal = statData.getDoubleStats().getLowValue(); minVal = minVal == null ? curVal : Math.min(minVal, curVal);
cs.setRange(csd.getLongStats().getLowValue(), csd.getLongStats().getHighValue()); } else if (colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) { cs.setCountDistint(csd.getDoubleStats().getNumDVs()); cs.setNumNulls(csd.getDoubleStats().getNumNulls()); cs.setAvgColLen(JavaDataModel.get().primitive1()); cs.setRange(csd.getDoubleStats().getLowValue(), csd.getDoubleStats().getHighValue()); } else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) { cs.setCountDistint(csd.getDoubleStats().getNumDVs()); cs.setNumNulls(csd.getDoubleStats().getNumNulls()); cs.setAvgColLen(JavaDataModel.get().primitive2()); cs.setRange(csd.getDoubleStats().getLowValue(), csd.getDoubleStats().getHighValue()); } else if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)
"", "", "", ""); } else if (csd.isSetDoubleStats()) { DoubleColumnStatsData dcsd = csd.getDoubleStats(); appendColumnStats(tableInfo, dcsd.getLowValue(), dcsd.getHighValue(), dcsd.getNumNulls(), dcsd.getNumDVs(), "", "", "", "");
cs.setRange(csd.getLongStats().getLowValue(), csd.getLongStats().getHighValue()); } else if (colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) { cs.setCountDistint(csd.getDoubleStats().getNumDVs()); cs.setNumNulls(csd.getDoubleStats().getNumNulls()); cs.setAvgColLen(JavaDataModel.get().primitive1()); cs.setRange(csd.getDoubleStats().getLowValue(), csd.getDoubleStats().getHighValue()); } else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) { cs.setCountDistint(csd.getDoubleStats().getNumDVs()); cs.setNumNulls(csd.getDoubleStats().getNumNulls()); cs.setAvgColLen(JavaDataModel.get().primitive2()); cs.setRange(csd.getDoubleStats().getLowValue(), csd.getDoubleStats().getHighValue()); } else if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)
DoubleColumnStatsData doubleStatsData = columnStatistics.getStatsData().getDoubleStats(); OptionalDouble min = doubleStatsData.isSetLowValue() ? OptionalDouble.of(doubleStatsData.getLowValue()) : OptionalDouble.empty(); OptionalDouble max = doubleStatsData.isSetHighValue() ? OptionalDouble.of(doubleStatsData.getHighValue()) : OptionalDouble.empty();
statsData.setDoubleStats(numericStats); statsData.getDoubleStats().setHighValue(highValue); statsData.getDoubleStats().setLowValue(lowValue); statsData.getDoubleStats().setNumDVs(numDVs); statsData.getDoubleStats().setNumNulls(numNulls); assertEquals(colStats2.getStatsData().getDoubleStats().getLowValue(), lowValue, 0.01); assertEquals(colStats2.getStatsData().getDoubleStats().getHighValue(), highValue, 0.01); assertEquals(colStats2.getStatsData().getDoubleStats().getNumNulls(), numNulls); assertEquals(colStats2.getStatsData().getDoubleStats().getNumDVs(), numDVs);
"", "", "", ""); } else if (csd.isSetDoubleStats()) { DoubleColumnStatsData dcsd = csd.getDoubleStats(); appendColumnStatsNoFormatting(colBuffer, dcsd.getLowValue(), dcsd.getHighValue(), dcsd.getNumNulls(), dcsd.getNumDVs(), "", "", "", "");
DoubleColumnStatsData oldDoubleStatsData = oldStatObj.getStatsData().getDoubleStats(); DoubleColumnStatsData newDoubleStatsData = newStatObj.getStatsData().getDoubleStats(); if (newDoubleStatsData.isSetHighValue()) { oldDoubleStatsData.setHighValue(newDoubleStatsData.getHighValue());
longStats.isSetHighValue() ? longStats.getHighValue() : null); } else if (statsObj.getStatsData().isSetDoubleStats()) { DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); mColStats.setDoubleStats( doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
longStats.isSetHighValue() ? longStats.getHighValue() : null); } else if (statsObj.getStatsData().isSetDoubleStats()) { DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); mColStats.setDoubleStats( doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,