private static ColumnStatistics booleanColumnStats(Long numberOfValues, Long trueValueCount) { BooleanStatistics booleanStatistics = null; if (trueValueCount != null) { booleanStatistics = new BooleanStatistics(trueValueCount); } return new ColumnStatistics(numberOfValues, booleanStatistics, null, null, null, null); }
@Override protected boolean chunkMatchesStats(List<Boolean> chunk, ColumnStatistics columnStatistics) { assertNull(columnStatistics.getIntegerStatistics()); assertNull(columnStatistics.getDoubleStatistics()); assertNull(columnStatistics.getStringStatistics()); assertNull(columnStatistics.getDateStatistics()); // check basic statistics if (!super.chunkMatchesStats(chunk, columnStatistics)) { return false; } // statistics can be missing for any reason if (columnStatistics.getBooleanStatistics() != null) { if (columnStatistics.getBooleanStatistics().getTrueValueCount() != Iterables.size(filter(chunk, equalTo(Boolean.TRUE)))) { return false; } } return true; } }
if (columnStatistics.hasNumberOfValues() && columnStatistics.getNumberOfValues() == 0) { return Domain.onlyNull(type); boolean hasNullValue = columnStatistics.getNumberOfValues() != rowCount; if (type.getJavaType() == boolean.class && columnStatistics.getBooleanStatistics() != null) { BooleanStatistics booleanStatistics = columnStatistics.getBooleanStatistics(); boolean hasFalseValues = (columnStatistics.getNumberOfValues() != booleanStatistics.getTrueValueCount()); if (hasTrueValues && hasFalseValues) { return Domain.all(BOOLEAN); else if (type.getTypeSignature().getBase().equals(StandardTypes.DATE) && columnStatistics.getDateStatistics() != null) { return createDomain(type, hasNullValue, columnStatistics.getDateStatistics(), value -> (long) value); else if (type.getJavaType() == long.class && columnStatistics.getIntegerStatistics() != null) { return createDomain(type, hasNullValue, columnStatistics.getIntegerStatistics()); else if (type.getJavaType() == double.class && columnStatistics.getDoubleStatistics() != null) { return createDomain(type, hasNullValue, columnStatistics.getDoubleStatistics()); else if (type.getJavaType() == Slice.class && columnStatistics.getStringStatistics() != null) { return createDomain(type, hasNullValue, columnStatistics.getStringStatistics());
return true; IntegerStatistics stats = statisticsByColumnIndex.get(0).getIntegerStatistics(); return (stats.getMin() == 50_000) || (stats.getMin() == 60_000); };
protected boolean chunkMatchesStats(List<T> chunk, ColumnStatistics columnStatistics) { // verify non null count if (columnStatistics.getNumberOfValues() != Iterables.size(filter(chunk, notNull()))) { return false; } return true; } }
assertTrue(columnStatistics.hasNumberOfValues());
return true; IntegerStatistics stats = statisticsByColumnIndex.get(0).getIntegerStatistics(); return ((stats.getMin() == 60) && (stats.getMax() == 117)) || ((stats.getMin() == 180) && (stats.getMax() == 237));
@Override protected boolean chunkMatchesStats(List<Long> chunk, ColumnStatistics columnStatistics) { assertNull(columnStatistics.getBooleanStatistics()); assertNull(columnStatistics.getIntegerStatistics()); assertNull(columnStatistics.getDoubleStatistics()); assertNull(columnStatistics.getStringStatistics()); // check basic statistics if (!super.chunkMatchesStats(chunk, columnStatistics)) { return false; } // statistics can be missing for any reason if (columnStatistics.getDateStatistics() != null) { // verify min Long min = columnStatistics.getDateStatistics().getMin().longValue(); if (!min.equals(Ordering.natural().nullsLast().min(chunk))) { return false; } // verify max Long statMax = columnStatistics.getDateStatistics().getMax().longValue(); Long chunkMax = Ordering.natural().nullsFirst().max(chunk); if (!statMax.equals(chunkMax)) { return false; } } return true; } }
private static ColumnStatistics dateColumnStats(Long numberOfValues, Integer minimum, Integer maximum) { return new ColumnStatistics(numberOfValues, null, null, null, null, new DateStatistics(minimum, maximum)); } }
@Override protected boolean chunkMatchesStats(List<Long> chunk, ColumnStatistics columnStatistics) { assertNull(columnStatistics.getBooleanStatistics()); assertNull(columnStatistics.getDoubleStatistics()); assertNull(columnStatistics.getStringStatistics()); assertNull(columnStatistics.getDateStatistics()); // check basic statistics if (!super.chunkMatchesStats(chunk, columnStatistics)) { return false; } // statistics can be missing for any reason if (columnStatistics.getIntegerStatistics() != null) { // verify min if (!columnStatistics.getIntegerStatistics().getMin().equals(Ordering.natural().nullsLast().min(chunk))) { return false; } // verify max if (!columnStatistics.getIntegerStatistics().getMax().equals(Ordering.natural().nullsFirst().max(chunk))) { return false; } } return true; } }
private static ColumnStatistics integerColumnStats(Long numberOfValues, Long minimum, Long maximum) { return new ColumnStatistics(numberOfValues, null, new IntegerStatistics(minimum, maximum), null, null, null); }
@Override protected boolean chunkMatchesStats(List<Double> chunk, ColumnStatistics columnStatistics) { assertNull(columnStatistics.getBooleanStatistics()); assertNull(columnStatistics.getIntegerStatistics()); assertNull(columnStatistics.getStringStatistics()); assertNull(columnStatistics.getDateStatistics()); // check basic statistics if (!super.chunkMatchesStats(chunk, columnStatistics)) { return false; } // statistics can be missing for any reason if (columnStatistics.getDoubleStatistics() != null) { // verify min if (!columnStatistics.getDoubleStatistics().getMin().equals(Ordering.natural().nullsLast().min(chunk))) { return false; } // verify max if (!columnStatistics.getDoubleStatistics().getMax().equals(Ordering.natural().nullsFirst().max(chunk))) { return false; } } return true; } }
private static ColumnStatistics doubleColumnStats(Long numberOfValues, Double minimum, Double maximum) { return new ColumnStatistics(numberOfValues, null, null, new DoubleStatistics(minimum, maximum), null, null); }
@Override protected boolean chunkMatchesStats(List<String> chunk, ColumnStatistics columnStatistics) assertNull(columnStatistics.getBooleanStatistics()); assertNull(columnStatistics.getIntegerStatistics()); assertNull(columnStatistics.getDoubleStatistics()); assertNull(columnStatistics.getDateStatistics()); if (columnStatistics.getStringStatistics() != null) { if (columnStatistics.getStringStatistics().getMin().compareTo(chunkMin) > 0) { return false; if (columnStatistics.getStringStatistics().getMax().compareTo(chunkMax) < 0) { return false;
private static ColumnStatistics toColumnStatistics(OrcProto.ColumnStatistics statistics, boolean isRowGroup) { return new ColumnStatistics( statistics.getNumberOfValues(), toBooleanStatistics(statistics.getBucketStatistics()), toIntegerStatistics(statistics.getIntStatistics()), toDoubleStatistics(statistics.getDoubleStatistics()), toStringStatistics(statistics.getStringStatistics(), isRowGroup), null); }
private static ColumnStatistics stringColumnStats(Long numberOfValues, String minimum, String maximum) { return new ColumnStatistics(numberOfValues, null, null, null, new StringStatistics(getMinSlice(minimum), getMaxSlice(maximum)), null); }
private static ColumnStatistics toColumnStatistics(OrcProto.ColumnStatistics statistics, boolean isRowGroup) { return new ColumnStatistics( statistics.getNumberOfValues(), toBooleanStatistics(statistics.getBucketStatistics()), toIntegerStatistics(statistics.getIntStatistics()), toDoubleStatistics(statistics.getDoubleStatistics()), toStringStatistics(statistics.getStringStatistics(), isRowGroup), toDateStatistics(statistics.getDateStatistics(), isRowGroup)); }