public ByteArrayOutputStream(CompressionKind compression, int bufferSize, StreamKind streamKind) { this.buffer = new OrcOutputBuffer(compression, bufferSize); this.streamKind = streamKind; }
@Override public void close() { closed = true; buffer.close(); }
@Override public void reset() { closed = false; buffer.reset(); checkpoints.clear(); } }
OrcOutputBuffer sliceOutput = new OrcOutputBuffer(CompressionKind.NONE, 3000); sliceOutput.writeBytes(largeByteArray, 0, 200); assertEquals(sliceOutput.getBufferCapacity(), 256); sliceOutput.writeBytes(largeByteArray, 0, 200); assertEquals(sliceOutput.getBufferCapacity(), 512); sliceOutput.writeBytes(largeByteArray, 0, 1200); assertEquals(sliceOutput.getBufferCapacity(), 1200); sliceOutput.writeBytes(largeByteArray, 0, 2000); assertEquals(sliceOutput.getBufferCapacity(), 2400); sliceOutput.writeBytes(largeByteArray, 0, 2500); assertEquals(sliceOutput.getBufferCapacity(), 3000); sliceOutput.close(); assertEquals(sliceOutput.writeDataTo(output), 200 + 200 + 1200 + 2000 + 2500);
@Test public void testWriteHugeByteChucks() { int size = 1024 * 1024; byte[] largeByteArray = new byte[size]; Arrays.fill(largeByteArray, (byte) 0xA); OrcOutputBuffer sliceOutput = new OrcOutputBuffer(CompressionKind.NONE, 256 * 1024); DynamicSliceOutput output = new DynamicSliceOutput(size); sliceOutput.writeBytes(largeByteArray, 10, size - 10); assertEquals(sliceOutput.writeDataTo(output), size - 10); assertEquals(output.slice(), wrappedBuffer(largeByteArray, 10, size - 10)); sliceOutput.reset(); output.reset(); sliceOutput.writeBytes(wrappedBuffer(largeByteArray), 100, size - 100); assertEquals(sliceOutput.writeDataTo(output), size - 100); assertEquals(output.slice(), wrappedBuffer(largeByteArray, 100, size - 100)); }
private Slice getSliceOutput() { buffer.close(); DynamicSliceOutput output = new DynamicSliceOutput(toIntExact(buffer.getOutputDataSize())); buffer.writeDataTo(output); Slice slice = output.slice(); buffer.reset(); return slice; } }
OrcOutputBuffer buffer = new OrcOutputBuffer(NONE, 1024); BooleanOutputStream output = new BooleanOutputStream(buffer); buffer.writeDataTo(slice); Slice batchWriteBuffer = slice.slice(); buffer.reset(); output.reset(); for (int count : counts) { List<BooleanStreamCheckpoint> singleWriteCheckpoints = output.getCheckpoints(); slice = new DynamicSliceOutput(128); buffer.writeDataTo(slice); Slice singleWriteBuffer = slice.slice();
@Override public SliceOutput appendBytes(byte[] source, int sourceIndex, int length) { writeBytes(source, sourceIndex, length); return this; }
@Override public void writeBytes(byte[] source, int sourceIndex, int length) { // Write huge chunks direct to OutputStream if (length >= DIRECT_FLUSH_SIZE) { // todo fill buffer before flushing flushBufferToOutputStream(); writeDirectlyToOutputStream(source, sourceIndex, length); bufferOffset += length; } else { ensureWritableBytes(length); slice.setBytes(bufferPosition, source, sourceIndex, length); bufferPosition += length; } }
@Override public long getBufferedBytes() { return buffer.estimateOutputDataSize(); }
@Override public void writeByte(int value) { ensureWritableBytes(SIZE_OF_BYTE); slice.setByte(bufferPosition, value); bufferPosition += SIZE_OF_BYTE; }
@Override public void close() { flushBufferToOutputStream(); }
@Override public void writeZero(int length) { checkArgument(length >= 0, "length must be 0 or greater than 0."); while (length > 0) { int batch = ensureBatchSize(length); Arrays.fill(buffer, bufferPosition, bufferPosition + batch, (byte) 0); bufferPosition += batch; length -= batch; } }
OrcOutputBuffer sliceOutput = new OrcOutputBuffer(CompressionKind.NONE, 3000); sliceOutput.writeBytes(largeByteArray, 0, 200); assertEquals(sliceOutput.getBufferCapacity(), 256); sliceOutput.writeBytes(largeByteArray, 0, 200); assertEquals(sliceOutput.getBufferCapacity(), 512); sliceOutput.writeBytes(largeByteArray, 0, 1200); assertEquals(sliceOutput.getBufferCapacity(), 1200); sliceOutput.writeBytes(largeByteArray, 0, 2000); assertEquals(sliceOutput.getBufferCapacity(), 2400); sliceOutput.writeBytes(largeByteArray, 0, 2500); assertEquals(sliceOutput.getBufferCapacity(), 3000); sliceOutput.close(); assertEquals(sliceOutput.writeDataTo(output), 200 + 200 + 1200 + 2000 + 2500);
@Test public void testWriteHugeByteChucks() { int size = 1024 * 1024; byte[] largeByteArray = new byte[size]; Arrays.fill(largeByteArray, (byte) 0xA); OrcOutputBuffer sliceOutput = new OrcOutputBuffer(CompressionKind.NONE, 256 * 1024); DynamicSliceOutput output = new DynamicSliceOutput(size); sliceOutput.writeBytes(largeByteArray, 10, size - 10); assertEquals(sliceOutput.writeDataTo(output), size - 10); assertEquals(output.slice(), wrappedBuffer(largeByteArray, 10, size - 10)); sliceOutput.reset(); output.reset(); sliceOutput.writeBytes(wrappedBuffer(largeByteArray), 100, size - 100); assertEquals(sliceOutput.writeDataTo(output), size - 100); assertEquals(output.slice(), wrappedBuffer(largeByteArray, 100, size - 100)); }
private Slice getSliceOutput() { buffer.close(); DynamicSliceOutput output = new DynamicSliceOutput(toIntExact(buffer.getOutputDataSize())); buffer.writeDataTo(output); Slice slice = output.slice(); buffer.reset(); return slice; } }
OrcOutputBuffer buffer = new OrcOutputBuffer(NONE, 1024); BooleanOutputStream output = new BooleanOutputStream(buffer); buffer.writeDataTo(slice); Slice batchWriteBuffer = slice.slice(); buffer.reset(); output.reset(); for (int count : counts) { List<BooleanStreamCheckpoint> singleWriteCheckpoints = output.getCheckpoints(); slice = new DynamicSliceOutput(128); buffer.writeDataTo(slice); Slice singleWriteBuffer = slice.slice();
@Override public SliceOutput appendBytes(byte[] source) { writeBytes(source); return this; }
@Override public void writeBytes(byte[] source, int sourceIndex, int length) { // Write huge chunks direct to OutputStream if (length >= DIRECT_FLUSH_SIZE) { // todo fill buffer before flushing flushBufferToOutputStream(); writeDirectlyToOutputStream(source, sourceIndex, length); bufferOffset += length; } else { ensureWritableBytes(length); slice.setBytes(bufferPosition, source, sourceIndex, length); bufferPosition += length; } }
@Override public long getBufferedBytes() { return buffer.estimateOutputDataSize() + size; }