private static OrcWriter createOrcFileWriter(OrcDataSink sink, List<Type> types) { List<String> columnNames = IntStream.range(0, types.size()) .mapToObj(String::valueOf) .collect(toImmutableList()); return new OrcWriter( sink, columnNames, types, ORC, LZ4, new OrcWriterOptions() .withMaxStringStatisticsLimit(new DataSize(0, BYTE)) .withStripeMinSize(new DataSize(64, MEGABYTE)) .withDictionaryMaxMemory(new DataSize(1, MEGABYTE)), ImmutableMap.of(), UTC, false, OrcWriteValidationMode.BOTH, new OrcWriterStats()); } }
public void recordStripeWritten(FlushReason flushReason, long stripeBytes, int stripeRows, int dictionaryBytes) { getFlushStats(flushReason).recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); allFlush.recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); }
closedStripesRetainedBytes += closedStripe.getRetainedSizeInBytes(); recordValidation(validation -> validation.addStripe(stripeInformation.getNumberOfRows())); stats.recordStripeWritten(flushReason, stripeInformation.getTotalLength(), stripeInformation.getNumberOfRows(), dictionaryCompressionOptimizer.getDictionaryMemoryBytes());
private void assertRoundTrip(Type writeType, Type readType, List<?> writeValues, List<?> readValues, boolean verifyWithHiveReader) throws Exception OrcWriterStats stats = new OrcWriterStats(); for (Format format : formats) { if (!format.supportsType(readType)) { assertEquals(stats.getWriterSizeInBytes(), 0);
@Override public void close() throws IOException { if (closed) { return; } closed = true; stats.updateSizeInBytes(-previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = 0; flushStripe(CLOSED); orcDataSink.close(); }
private void assertRoundTrip(Type writeType, Type readType, List<?> writeValues, List<?> readValues, boolean verifyWithHiveReader) throws Exception OrcWriterStats stats = new OrcWriterStats(); for (Format format : formats) { if (!format.supportsType(readType)) { assertEquals(stats.getWriterSizeInBytes(), 0);
stats.updateSizeInBytes(previouslyRecordedSizeInBytes);
public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, ORC, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
stats.updateSizeInBytes(recordedSizeInBytes - previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = recordedSizeInBytes;
public void recordStripeWritten(FlushReason flushReason, long stripeBytes, int stripeRows, int dictionaryBytes) { getFlushStats(flushReason).recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); allFlush.recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); }
closedStripesRetainedBytes += closedStripe.getRetainedSizeInBytes(); recordValidation(validation -> validation.addStripe(stripeInformation.getNumberOfRows())); stats.recordStripeWritten(flushReason, stripeInformation.getTotalLength(), stripeInformation.getNumberOfRows(), dictionaryCompressionOptimizer.getDictionaryMemoryBytes());
public PrestoDwrfFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, DWRF, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
@Override public void close() throws IOException { if (closed) { return; } closed = true; stats.updateSizeInBytes(-previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = 0; flushStripe(CLOSED); orcDataSink.close(); }
true, BOTH, new OrcWriterStats());
stats.updateSizeInBytes(previouslyRecordedSizeInBytes);
true, validationMode, new OrcWriterStats());
stats.updateSizeInBytes(recordedSizeInBytes - previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = recordedSizeInBytes;
public PrestoDwrfFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, DWRF, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, ORC, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
true, BOTH, new OrcWriterStats());