OrcWriterStats stats) this.validationBuilder = validate ? new OrcWriteValidationBuilder(validationMode, types) .setStringStatisticsLimitInBytes(toIntExact(options.getMaxStringStatisticsLimit().toBytes())) : null; this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.compression = requireNonNull(compression, "compression is null"); recordValidation(validation -> validation.setCompression(compression)); this.stripeMaxRowCount = options.getStripeMaxRowCount(); this.rowGroupMaxRowCount = options.getRowGroupMaxRowCount(); recordValidation(validation -> validation.setRowGroupMaxRowCount(rowGroupMaxRowCount)); this.maxCompressionBufferSize = toIntExact(options.getMaxCompressionBufferSize().toBytes()); recordValidation(validation -> validation.setColumnNames(columnNames)); recordValidation(validation -> validation.addMetadataProperty(entry.getKey(), utf8Slice(entry.getValue())));
private void finishRowGroup() { Map<Integer, ColumnStatistics> columnStatistics = new HashMap<>(); columnWriters.forEach(columnWriter -> columnStatistics.putAll(columnWriter.finishRowGroup())); recordValidation(validation -> validation.addRowGroupStatistics(columnStatistics)); rowGroupRowCount = 0; }
recordValidation(validation -> validation.addStripeStatistics(stripeStartOffset, statistics)); StripeInformation stripeInformation = new StripeInformation(stripeRowCount, stripeStartOffset, indexLength, dataLength, footer.length()); ClosedStripe closedStripe = new ClosedStripe(stripeInformation, statistics); closedStripes.add(closedStripe); closedStripesRetainedBytes += closedStripe.getRetainedSizeInBytes(); recordValidation(validation -> validation.addStripe(stripeInformation.getNumberOfRows())); stats.recordStripeWritten(flushReason, stripeInformation.getTotalLength(), stripeInformation.getNumberOfRows(), dictionaryCompressionOptimizer.getDictionaryMemoryBytes());
OrcWriterStats stats) this.validationBuilder = validate ? new OrcWriteValidation.OrcWriteValidationBuilder(validationMode, types).setStringStatisticsLimitInBytes(toIntExact(options.getMaxStringStatisticsLimit().toBytes())) : null; this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.compression = requireNonNull(compression, "compression is null"); recordValidation(validation -> validation.setCompression(compression)); this.stripeMaxRowCount = options.getStripeMaxRowCount(); this.rowGroupMaxRowCount = options.getRowGroupMaxRowCount(); recordValidation(validation -> validation.setRowGroupMaxRowCount(rowGroupMaxRowCount)); this.maxCompressionBufferSize = toIntExact(options.getMaxCompressionBufferSize().toBytes()); recordValidation(validation -> validation.setColumnNames(columnNames)); recordValidation(validation -> validation.addMetadataProperty(entry.getKey(), utf8Slice(entry.getValue())));
recordValidation(validation -> validation.addStripeStatistics(stripeStartOffset, statistics)); StripeInformation stripeInformation = new StripeInformation(stripeRowCount, stripeStartOffset, indexLength, dataLength, footer.length()); ClosedStripe closedStripe = new ClosedStripe(stripeInformation, statistics); closedStripes.add(closedStripe); closedStripesRetainedBytes += closedStripe.getRetainedSizeInBytes(); recordValidation(validation -> validation.addStripe(stripeInformation.getNumberOfRows())); stats.recordStripeWritten(flushReason, stripeInformation.getTotalLength(), stripeInformation.getNumberOfRows(), dictionaryCompressionOptimizer.getDictionaryMemoryBytes());
validationBuilder.addPage(page);
validationBuilder.addPage(page);
public void validate(OrcDataSource input) throws OrcCorruptionException { checkState(validationBuilder != null, "validation is not enabled"); validateFile( validationBuilder.build(), input, types, hiveStorageTimeZone, orcEncoding); }
public long getRetainedBytes() { return INSTANCE_SIZE + columnWritersRetainedBytes + closedStripesRetainedBytes + orcDataSink.getRetainedSizeInBytes() + (validationBuilder == null ? 0 : validationBuilder.getRetainedSize()); }
public long getRetainedBytes() { return INSTANCE_SIZE + columnWritersRetainedBytes + closedStripesRetainedBytes + orcDataSink.getRetainedSizeInBytes() + (validationBuilder == null ? 0 : validationBuilder.getRetainedSize()); }
public void validate(OrcDataSource input) throws OrcCorruptionException { checkState(validationBuilder != null, "validation is not enabled"); validateFile( validationBuilder.build(), input, types, hiveStorageTimeZone, orcEncoding); }
private void finishRowGroup() { Map<Integer, ColumnStatistics> columnStatistics = new HashMap<>(); columnWriters.forEach(columnWriter -> columnStatistics.putAll(columnWriter.finishRowGroup())); recordValidation(validation -> validation.addRowGroupStatistics(columnStatistics)); rowGroupRowCount = 0; }