protected CompressedRandomAccessReader(String dataFilePath, CompressionMetadata metadata, PoolingSegmentedFile owner) throws FileNotFoundException { super(new File(dataFilePath), metadata.chunkLength(), metadata.compressedFileLength, owner); this.metadata = metadata; checksum = metadata.hasPostCompressionAdlerChecksums ? new Adler32() : new CRC32(); compressed = ByteBuffer.wrap(new byte[metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())]); }
/** * Compress the raw data, as well as manage sizing of the {@code outputBuffer}; if the buffer is not big enough, * deallocate current, and allocate a large enough buffer. * Write the two header lengths (plain text length, compressed length) to the beginning of the buffer as we want those * values encapsulated in the encrypted block, as well. * * @return the byte buffer that was actaully written to; it may be the {@code outputBuffer} if it had enough capacity, * or it may be a new, larger instance. Callers should capture the return buffer (if calling multiple times). */ public static ByteBuffer compress(ByteBuffer inputBuffer, ByteBuffer outputBuffer, boolean allowBufferResize, ICompressor compressor) throws IOException { int inputLength = inputBuffer.remaining(); final int compressedLength = compressor.initialCompressedBufferLength(inputLength); outputBuffer = ByteBufferUtil.ensureCapacity(outputBuffer, compressedLength + COMPRESSED_BLOCK_HEADER_SIZE, allowBufferResize); outputBuffer.putInt(inputLength); compressor.compress(inputBuffer, outputBuffer); outputBuffer.flip(); return outputBuffer; }
/** * Compress the raw data, as well as manage sizing of the {@code outputBuffer}; if the buffer is not big enough, * deallocate current, and allocate a large enough buffer. * Write the two header lengths (plain text length, compressed length) to the beginning of the buffer as we want those * values encapsulated in the encrypted block, as well. * * @return the byte buffer that was actaully written to; it may be the {@code outputBuffer} if it had enough capacity, * or it may be a new, larger instance. Callers should capture the return buffer (if calling multiple times). */ public static ByteBuffer compress(ByteBuffer inputBuffer, ByteBuffer outputBuffer, boolean allowBufferResize, ICompressor compressor) throws IOException { int inputLength = inputBuffer.remaining(); final int compressedLength = compressor.initialCompressedBufferLength(inputLength); outputBuffer = ByteBufferUtil.ensureCapacity(outputBuffer, compressedLength + COMPRESSED_BLOCK_HEADER_SIZE, allowBufferResize); outputBuffer.putInt(inputLength); compressor.compress(inputBuffer, outputBuffer); outputBuffer.flip(); return outputBuffer; }
/** * Compress the raw data, as well as manage sizing of the {@code outputBuffer}; if the buffer is not big enough, * deallocate current, and allocate a large enough buffer. * Write the two header lengths (plain text length, compressed length) to the beginning of the buffer as we want those * values encapsulated in the encrypted block, as well. * * @return the byte buffer that was actaully written to; it may be the {@code outputBuffer} if it had enough capacity, * or it may be a new, larger instance. Callers should capture the return buffer (if calling multiple times). */ public static ByteBuffer compress(ByteBuffer inputBuffer, ByteBuffer outputBuffer, boolean allowBufferResize, ICompressor compressor) throws IOException { int inputLength = inputBuffer.remaining(); final int compressedLength = compressor.initialCompressedBufferLength(inputLength); outputBuffer = ByteBufferUtil.ensureCapacity(outputBuffer, compressedLength + COMPRESSED_BLOCK_HEADER_SIZE, allowBufferResize); outputBuffer.putInt(inputLength); compressor.compress(inputBuffer, outputBuffer); outputBuffer.flip(); return outputBuffer; }
protected void writeBuffer(ByteBuffer bb) throws IOException { int originalSize = bb.remaining(); int estimatedSize = compressor.initialCompressedBufferLength(originalSize) + METADATA_SIZE; if (compressionBuffer == null || compressionBuffer.capacity() < estimatedSize) { compressionBuffer = compressor.preferredBufferType().allocate(estimatedSize); } compressionBuffer.clear(); compressionBuffer.position(METADATA_SIZE); compressor.compress(bb, compressionBuffer); int compressedSize = compressionBuffer.position() - METADATA_SIZE; compressionBuffer.rewind(); compressionBuffer.putInt(originalSize); compressionBuffer.putInt(compressedSize); compressionBuffer.rewind(); compressionBuffer.limit(compressedSize + METADATA_SIZE); super.writeBuffer(compressionBuffer); }
protected void writeBuffer(ByteBuffer bb) throws IOException { int originalSize = bb.remaining(); int estimatedSize = compressor.initialCompressedBufferLength(originalSize) + METADATA_SIZE; if (compressionBuffer == null || compressionBuffer.capacity() < estimatedSize) { compressionBuffer = compressor.preferredBufferType().allocate(estimatedSize); } compressionBuffer.clear(); compressionBuffer.position(METADATA_SIZE); compressor.compress(bb, compressionBuffer); int compressedSize = compressionBuffer.position() - METADATA_SIZE; compressionBuffer.rewind(); compressionBuffer.putInt(originalSize); compressionBuffer.putInt(compressedSize); compressionBuffer.rewind(); compressionBuffer.limit(compressedSize + METADATA_SIZE); super.writeBuffer(compressionBuffer); }
protected void writeBuffer(ByteBuffer bb) throws IOException { int originalSize = bb.remaining(); int estimatedSize = compressor.initialCompressedBufferLength(originalSize) + METADATA_SIZE; if (compressionBuffer == null || compressionBuffer.capacity() < estimatedSize) { compressionBuffer = compressor.preferredBufferType().allocate(estimatedSize); } compressionBuffer.clear(); compressionBuffer.position(METADATA_SIZE); compressor.compress(bb, compressionBuffer); int compressedSize = compressionBuffer.position() - METADATA_SIZE; compressionBuffer.rewind(); compressionBuffer.putInt(originalSize); compressionBuffer.putInt(compressedSize); compressionBuffer.rewind(); compressionBuffer.limit(compressedSize + METADATA_SIZE); super.writeBuffer(compressionBuffer); }
@Override void write(int startMarker, int nextMarker) { int contentStart = startMarker + SYNC_MARKER_SIZE; int length = nextMarker - contentStart; // The length may be 0 when the segment is being closed. assert length > 0 || length == 0 && !isStillAllocating(); try { int neededBufferSize = compressor.initialCompressedBufferLength(length) + COMPRESSED_MARKER_SIZE; ByteBuffer compressedBuffer = manager.getBufferPool().getThreadLocalReusableBuffer(neededBufferSize); ByteBuffer inputBuffer = buffer.duplicate(); inputBuffer.limit(contentStart + length).position(contentStart); compressedBuffer.limit(compressedBuffer.capacity()).position(COMPRESSED_MARKER_SIZE); compressor.compress(inputBuffer, compressedBuffer); compressedBuffer.flip(); compressedBuffer.putInt(SYNC_MARKER_SIZE, length); // Only one thread can be here at a given time. // Protected by synchronization on CommitLogSegment.sync(). writeSyncMarker(id, compressedBuffer, 0, (int) channel.position(), (int) channel.position() + compressedBuffer.remaining()); manager.addSize(compressedBuffer.limit()); channel.write(compressedBuffer); assert channel.position() - lastWrittenPos == compressedBuffer.limit(); lastWrittenPos = channel.position(); } catch (Exception e) { throw new FSWriteError(e, getPath()); } }
@Override void write(int startMarker, int nextMarker) { int contentStart = startMarker + SYNC_MARKER_SIZE; int length = nextMarker - contentStart; // The length may be 0 when the segment is being closed. assert length > 0 || length == 0 && !isStillAllocating(); try { int neededBufferSize = compressor.initialCompressedBufferLength(length) + COMPRESSED_MARKER_SIZE; ByteBuffer compressedBuffer = manager.getBufferPool().getThreadLocalReusableBuffer(neededBufferSize); ByteBuffer inputBuffer = buffer.duplicate(); inputBuffer.limit(contentStart + length).position(contentStart); compressedBuffer.limit(compressedBuffer.capacity()).position(COMPRESSED_MARKER_SIZE); compressor.compress(inputBuffer, compressedBuffer); compressedBuffer.flip(); compressedBuffer.putInt(SYNC_MARKER_SIZE, length); // Only one thread can be here at a given time. // Protected by synchronization on CommitLogSegment.sync(). writeSyncMarker(id, compressedBuffer, 0, (int) channel.position(), (int) channel.position() + compressedBuffer.remaining()); manager.addSize(compressedBuffer.limit()); channel.write(compressedBuffer); assert channel.position() - lastWrittenPos == compressedBuffer.limit(); lastWrittenPos = channel.position(); } catch (Exception e) { throw new FSWriteError(e, getPath()); } }
public ByteBuffer allocateBuffer() { return allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())); }
@Override void write(int startMarker, int nextMarker) { int contentStart = startMarker + SYNC_MARKER_SIZE; int length = nextMarker - contentStart; // The length may be 0 when the segment is being closed. assert length > 0 || length == 0 && !isStillAllocating(); try { int neededBufferSize = compressor.initialCompressedBufferLength(length) + COMPRESSED_MARKER_SIZE; ByteBuffer compressedBuffer = manager.getBufferPool().getThreadLocalReusableBuffer(neededBufferSize); ByteBuffer inputBuffer = buffer.duplicate(); inputBuffer.limit(contentStart + length).position(contentStart); compressedBuffer.limit(compressedBuffer.capacity()).position(COMPRESSED_MARKER_SIZE); compressor.compress(inputBuffer, compressedBuffer); compressedBuffer.flip(); compressedBuffer.putInt(SYNC_MARKER_SIZE, length); // Only one thread can be here at a given time. // Protected by synchronization on CommitLogSegment.sync(). writeSyncMarker(id, compressedBuffer, 0, (int) channel.position(), (int) channel.position() + compressedBuffer.remaining()); manager.addSize(compressedBuffer.limit()); channel.write(compressedBuffer); assert channel.position() - lastWrittenPos == compressedBuffer.limit(); lastWrittenPos = channel.position(); } catch (Exception e) { throw new FSWriteError(e, getPath()); } }
public ByteBuffer allocateBuffer() { return allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())); }
public ByteBuffer allocateBuffer() { return allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())); }
public ByteBuffer allocateBuffer() { return allocateBuffer(metadata.compressor().initialCompressedBufferLength(metadata.chunkLength())); }
public CompressedSequentialWriter(File file, String offsetsPath, CompressionParameters parameters, MetadataCollector sstableMetadataCollector) { super(file, parameters.chunkLength()); this.compressor = parameters.sstableCompressor; // buffer for compression should be the same size as buffer itself compressed = new ICompressor.WrappedArray(new byte[compressor.initialCompressedBufferLength(buffer.length)]); /* Index File (-CompressionInfo.db component) and it's header */ metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsPath); this.sstableMetadataCollector = sstableMetadataCollector; crcMetadata = new DataIntegrityMetadata.ChecksumWriter(out); }
compressed = compressor.preferredBufferType().allocate(compressor.initialCompressedBufferLength(buffer.capacity()));
compressed = compressor.preferredBufferType().allocate(compressor.initialCompressedBufferLength(buffer.capacity()));
compressed = compressor.preferredBufferType().allocate(compressor.initialCompressedBufferLength(buffer.capacity()));
compressed = compressor.preferredBufferType().allocate(compressor.initialCompressedBufferLength(buffer.capacity()));