@Test public void testSingleDistinctValue() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(10)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(0)); doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(11)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(1)); }
doubleStats.setNumNulls(Long.parseLong(value)); } else if (fName.equals("numDVs")) { doubleStats.setNumDVs(Long.parseLong(value)); } else if (fName.equals("lowValue")) { doubleStats.setLowValue(Double.parseDouble(value));
private static void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setLowValue(d); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(buf); ; } }
@Test public void testDoubleStatsToColumnStatistics() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setLowValue(0); doubleColumnStatsData.setHighValue(100); doubleColumnStatsData.setNumNulls(1); doubleColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.empty()); assertEquals(actual.getDoubleStatistics(), Optional.of(new DoubleStatistics(OptionalDouble.of(0), OptionalDouble.of(100)))); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setLowValue(d); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(v);; } }
aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues()); ColumnStatisticsData csd = new ColumnStatisticsData(); csd.setDoubleStats(aggregateData); aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues()); ColumnStatisticsData csd = new ColumnStatisticsData(); csd.setDoubleStats(aggregateData);
unsetNumDVs(); } else { setNumDVs((Long)value);
statsData.getDoubleStats().setNumDVs(numDVs); statsData.getDoubleStats().setNumNulls(numNulls);
oldDoubleStatsData.setNumDVs(newDoubleStatsData.getNumDVs());
private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setLowValue(d); } }
@Test public void testSingleDistinctValue() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(10)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(0)); doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(11)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(1)); }
@Test public void testSingleDistinctValue() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(10)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(0)); doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(11)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(1)); }
@Test public void testSingleDistinctValue() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(10)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(0)); doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setNumNulls(10); doubleColumnStatsData.setNumDVs(1); columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(11)); assertEquals(actual.getNullsCount(), OptionalLong.of(10)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(1)); }
@Test public void testDoubleStatsToColumnStatistics() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setLowValue(0); doubleColumnStatsData.setHighValue(100); doubleColumnStatsData.setNumNulls(1); doubleColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.empty()); assertEquals(actual.getDoubleStatistics(), Optional.of(new DoubleStatistics(OptionalDouble.of(0), OptionalDouble.of(100)))); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
@Test public void testDoubleStatsToColumnStatistics() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setLowValue(0); doubleColumnStatsData.setHighValue(100); doubleColumnStatsData.setNumNulls(1); doubleColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.empty()); assertEquals(actual.getDoubleStatistics(), Optional.of(new DoubleStatistics(OptionalDouble.of(0), OptionalDouble.of(100)))); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
@Test public void testDoubleStatsToColumnStatistics() { DoubleColumnStatsData doubleColumnStatsData = new DoubleColumnStatsData(); doubleColumnStatsData.setLowValue(0); doubleColumnStatsData.setHighValue(100); doubleColumnStatsData.setNumNulls(1); doubleColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DOUBLE_TYPE_NAME, doubleStats(doubleColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.empty()); assertEquals(actual.getDoubleStatistics(), Optional.of(new DoubleStatistics(OptionalDouble.of(0), OptionalDouble.of(100)))); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
unsetNumDVs(); } else { setNumDVs((Long)value);
unsetNumDVs(); } else { setNumDVs((Long)value);
unsetNumDVs(); } else { setNumDVs((Long)value);
doubleStats.setHighValue(mStatsObj.getDoubleHighValue()); doubleStats.setLowValue(mStatsObj.getDoubleLowValue()); doubleStats.setNumDVs(mStatsObj.getNumDVs()); colStatsData.setDoubleStats(doubleStats);