.withMin(minBytes) .withMax(maxBytes) .withNumNulls(numNulls) .build(); break;
byte[] max = formatStats.max_value.array(); if (isMinMaxStatsSupported(type) || Arrays.equals(min, max)) { statsBuilder.withMin(min); statsBuilder.withMax(max); statsBuilder.withNumNulls(formatStats.null_count); (sortOrdersMatch || maxEqualsMin)) { if (isSet) { statsBuilder.withMin(formatStats.min.array()); statsBuilder.withMax(formatStats.max.array()); statsBuilder.withNumNulls(formatStats.null_count); return statsBuilder.build();
byte[] max = formatStats.max_value.array(); if (isMinMaxStatsSupported(type) || Arrays.equals(min, max)) { statsBuilder.withMin(min); statsBuilder.withMax(max); (sortOrdersMatch || maxEqualsMin)) { if (isSet) { statsBuilder.withMin(formatStats.min.array()); statsBuilder.withMax(formatStats.max.array()); statsBuilder.withNumNulls(formatStats.null_count); return statsBuilder.build();
@Override public Statistics<?> build() { DoubleStatistics stats = (DoubleStatistics) super.build(); if (stats.hasNonNullValue()) { Double min = stats.genericGetMin(); Double max = stats.genericGetMax(); // Drop min/max values in case of NaN as the sorting order of values is undefined for this case if (min.isNaN() || max.isNaN()) { stats.setMinMax(0.0, 0.0); ((Statistics<?>) stats).hasNonNullValue = false; } else { // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped if (Double.compare(min, 0.0) == 0) { min = -0.0; stats.setMinMax(min, max); } if (Double.compare(max, -0.0) == 0) { max = 0.0; stats.setMinMax(min, max); } } } return stats; } }
@Override public Statistics<?> build() { DoubleStatistics stats = (DoubleStatistics) super.build(); if (stats.hasNonNullValue()) { Double min = stats.genericGetMin(); Double max = stats.genericGetMax(); // Drop min/max values in case of NaN as the sorting order of values is undefined for this case if (min.isNaN() || max.isNaN()) { stats.setMinMax(0.0, 0.0); ((Statistics<?>) stats).hasNonNullValue = false; } else { // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped if (Double.compare(min, 0.0) == 0) { min = -0.0; stats.setMinMax(min, max); } if (Double.compare(max, -0.0) == 0) { max = 0.0; stats.setMinMax(min, max); } } } return stats; } }
@Override public Statistics<?> build() { FloatStatistics stats = (FloatStatistics) super.build(); if (stats.hasNonNullValue()) { Float min = stats.genericGetMin(); Float max = stats.genericGetMax(); // Drop min/max values in case of NaN as the sorting order of values is undefined for this case if (min.isNaN() || max.isNaN()) { stats.setMinMax(0.0f, 0.0f); ((Statistics<?>) stats).hasNonNullValue = false; } else { // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped if (Float.compare(min, 0.0f) == 0) { min = -0.0f; stats.setMinMax(min, max); } if (Float.compare(max, -0.0f) == 0) { max = 0.0f; stats.setMinMax(min, max); } } } return stats; } }
@Override public Statistics<?> build() { FloatStatistics stats = (FloatStatistics) super.build(); if (stats.hasNonNullValue()) { Float min = stats.genericGetMin(); Float max = stats.genericGetMax(); // Drop min/max values in case of NaN as the sorting order of values is undefined for this case if (min.isNaN() || max.isNaN()) { stats.setMinMax(0.0f, 0.0f); ((Statistics<?>) stats).hasNonNullValue = false; } else { // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped if (Float.compare(min, 0.0f) == 0) { min = -0.0f; stats.setMinMax(min, max); } if (Float.compare(max, -0.0f) == 0) { max = 0.0f; stats.setMinMax(min, max); } } } return stats; } }
/** * Creates decimal statistics where min and max values are re-created using given scale. * * @param statistics statistics that needs to be adjusted * @param scale adjustment scale * @return adjusted statistics */ @SuppressWarnings("unchecked") private Statistics<C> adjustDecimalStatistics(Statistics<C> statistics, int scale) { byte[] minBytes = new BigDecimal(new BigInteger(statistics.getMinBytes())) .setScale(scale, RoundingMode.HALF_UP).unscaledValue().toByteArray(); byte[] maxBytes = new BigDecimal(new BigInteger(statistics.getMaxBytes())) .setScale(scale, RoundingMode.HALF_UP).unscaledValue().toByteArray(); return (Statistics<C>) Statistics.getBuilderForReading(statistics.type()) .withMin(minBytes) .withMax(maxBytes) .withNumNulls(statistics.getNumNulls()) .build(); }
/** * Returns a builder to create new statistics object. Used to read the statistics from the parquet file. * * @param type * type of the column * @return builder to create new statistics object */ public static Builder getBuilderForReading(PrimitiveType type) { switch (type.getPrimitiveTypeName()) { case FLOAT: return new FloatBuilder(type); case DOUBLE: return new DoubleBuilder(type); default: return new Builder(type); } }
/** * Returns a builder to create new statistics object. Used to read the statistics from the parquet file. * * @param type * type of the column * @return builder to create new statistics object */ public static Builder getBuilderForReading(PrimitiveType type) { switch (type.getPrimitiveTypeName()) { case FLOAT: return new FloatBuilder(type); case DOUBLE: return new DoubleBuilder(type); default: return new Builder(type); } }
@SuppressWarnings("unchecked") private Statistics<T> getStatistics(BigDecimal min, BigDecimal max, DecimalMetadata decimalMetadata) { PrimitiveType decimalType = org.apache.parquet.schema.Types.optional(PrimitiveType.PrimitiveTypeName.BINARY) .as(OriginalType.DECIMAL) .precision(decimalMetadata.getPrecision()) .scale(decimalMetadata.getScale()) .named("decimal_type"); return (Statistics<T>) Statistics.getBuilderForReading(decimalType) .withMin(min.unscaledValue().toByteArray()) .withMax(max.unscaledValue().toByteArray()) .withNumNulls(0) .build(); }