private void finishRowGroup() { Map<Integer, ColumnStatistics> columnStatistics = new HashMap<>(); columnWriters.forEach(columnWriter -> columnStatistics.putAll(columnWriter.finishRowGroup())); recordValidation(validation -> validation.addRowGroupStatistics(columnStatistics)); rowGroupRowCount = 0; }
private void finishRowGroup() { Map<Integer, ColumnStatistics> columnStatistics = new HashMap<>(); columnWriters.forEach(columnWriter -> columnStatistics.putAll(columnWriter.finishRowGroup())); recordValidation(validation -> validation.addRowGroupStatistics(columnStatistics)); rowGroupRowCount = 0; }
.map(StripeStatistics::getColumnStatistics) .collect(toList())); recordValidation(validation -> validation.setFileStatistics(fileStats)); outputData.add(createDataOutput(footerSlice)); recordValidation(validation -> validation.setVersion(metadataWriter.getOrcMetadataVersion())); Slice postscriptSlice = metadataWriter.writePostscript(footerSlice.length(), metadataSlice.length(), compression, maxCompressionBufferSize); outputData.add(createDataOutput(postscriptSlice));
.map(StripeStatistics::getColumnStatistics) .collect(toList())); recordValidation(validation -> validation.setFileStatistics(fileStats)); outputData.add(createDataOutput(footerSlice)); recordValidation(validation -> validation.setVersion(metadataWriter.getOrcMetadataVersion())); Slice postscriptSlice = metadataWriter.writePostscript(footerSlice.length(), metadataSlice.length(), compression, maxCompressionBufferSize); outputData.add(createDataOutput(postscriptSlice));
this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.compression = requireNonNull(compression, "compression is null"); recordValidation(validation -> validation.setCompression(compression)); this.stripeMaxRowCount = options.getStripeMaxRowCount(); this.rowGroupMaxRowCount = options.getRowGroupMaxRowCount(); recordValidation(validation -> validation.setRowGroupMaxRowCount(rowGroupMaxRowCount)); this.maxCompressionBufferSize = toIntExact(options.getMaxCompressionBufferSize().toBytes()); recordValidation(validation -> validation.setColumnNames(columnNames)); recordValidation(validation -> validation.addMetadataProperty(entry.getKey(), utf8Slice(entry.getValue())));
this.orcEncoding = requireNonNull(orcEncoding, "orcEncoding is null"); this.compression = requireNonNull(compression, "compression is null"); recordValidation(validation -> validation.setCompression(compression)); this.stripeMaxRowCount = options.getStripeMaxRowCount(); this.rowGroupMaxRowCount = options.getRowGroupMaxRowCount(); recordValidation(validation -> validation.setRowGroupMaxRowCount(rowGroupMaxRowCount)); this.maxCompressionBufferSize = toIntExact(options.getMaxCompressionBufferSize().toBytes()); recordValidation(validation -> validation.setColumnNames(columnNames)); recordValidation(validation -> validation.addMetadataProperty(entry.getKey(), utf8Slice(entry.getValue())));
recordValidation(validation -> validation.addStripeStatistics(stripeStartOffset, statistics)); StripeInformation stripeInformation = new StripeInformation(stripeRowCount, stripeStartOffset, indexLength, dataLength, footer.length()); ClosedStripe closedStripe = new ClosedStripe(stripeInformation, statistics); closedStripes.add(closedStripe); closedStripesRetainedBytes += closedStripe.getRetainedSizeInBytes(); recordValidation(validation -> validation.addStripe(stripeInformation.getNumberOfRows())); stats.recordStripeWritten(flushReason, stripeInformation.getTotalLength(), stripeInformation.getNumberOfRows(), dictionaryCompressionOptimizer.getDictionaryMemoryBytes());
recordValidation(validation -> validation.addStripeStatistics(stripeStartOffset, statistics)); StripeInformation stripeInformation = new StripeInformation(stripeRowCount, stripeStartOffset, indexLength, dataLength, footer.length()); ClosedStripe closedStripe = new ClosedStripe(stripeInformation, statistics); closedStripes.add(closedStripe); closedStripesRetainedBytes += closedStripe.getRetainedSizeInBytes(); recordValidation(validation -> validation.addStripe(stripeInformation.getNumberOfRows())); stats.recordStripeWritten(flushReason, stripeInformation.getTotalLength(), stripeInformation.getNumberOfRows(), dictionaryCompressionOptimizer.getDictionaryMemoryBytes());