@Override public void close() throws IOException { if (closed) { return; } closed = true; stats.updateSizeInBytes(-previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = 0; flushStripe(CLOSED); orcDataSink.close(); }
public void recordStripeWritten(FlushReason flushReason, long stripeBytes, int stripeRows, int dictionaryBytes) { getFlushStats(flushReason).recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); allFlush.recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); }
public long getRetainedBytes() { return INSTANCE_SIZE + columnWritersRetainedBytes + closedStripesRetainedBytes + orcDataSink.getRetainedSizeInBytes() + (validationBuilder == null ? 0 : validationBuilder.getRetainedSize()); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
public long getBufferedBytes() { if (direct) { return (long) (rowCount * valuesPerRow * bytesPerEntry); } int dictionaryEntries = getDictionaryEntries(); int bytesPerValue = estimateIndexBytesPerValue(dictionaryEntries); return (dictionaryEntries * bytesPerEntry) + (getNonNullValueCount() * bytesPerValue); }
public boolean isDictionaryMemoryFull() { return optimizer.isFull(getBufferedBytes()); }
private <K, V> void runTest(String testOrcFileName, Type keyType, Type valueType, ExpectedValuesBuilder<K, V> expectedValuesBuilder) throws Exception { List<Map<K, V>> expectedValues = expectedValuesBuilder.build(); runTest(testOrcFileName, keyType, valueType, expectedValues, false, false); runTest(testOrcFileName, keyType, valueType, expectedValues, true, false); runTest(testOrcFileName, keyType, valueType, expectedValues, false, true); }
@Override public final String toString() { return id.toString(); }
@Override public SliceOutput appendBytes(byte[] source, int sourceIndex, int length) { writeBytes(source, sourceIndex, length); return this; }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
public long getBufferedBytes() { if (direct) { return (long) (rowCount * valuesPerRow * bytesPerEntry); } int dictionaryEntries = getDictionaryEntries(); int bytesPerValue = estimateIndexBytesPerValue(dictionaryEntries); return (dictionaryEntries * bytesPerEntry) + (getNonNullValueCount() * bytesPerValue); }
@Override public void close() throws IOException { if (closed) { return; } closed = true; stats.updateSizeInBytes(-previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = 0; flushStripe(CLOSED); orcDataSink.close(); }
public boolean isDictionaryMemoryFull() { return optimizer.isFull(getBufferedBytes()); }
private <K, V> void runTest(String testOrcFileName, Type keyType, Type valueType, ExpectedValuesBuilder<K, V> expectedValuesBuilder) throws Exception { List<Map<K, V>> expectedValues = expectedValuesBuilder.build(); runTest(testOrcFileName, keyType, valueType, expectedValues, false, false); runTest(testOrcFileName, keyType, valueType, expectedValues, true, false); runTest(testOrcFileName, keyType, valueType, expectedValues, false, true); }
public void recordStripeWritten(FlushReason flushReason, long stripeBytes, int stripeRows, int dictionaryBytes) { getFlushStats(flushReason).recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); allFlush.recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); }
public long getRetainedBytes() { return INSTANCE_SIZE + columnWritersRetainedBytes + closedStripesRetainedBytes + orcDataSink.getRetainedSizeInBytes() + (validationBuilder == null ? 0 : validationBuilder.getRetainedSize()); }