public int chunkLength() { return parameters.chunkLength(); }
private String chunkLengthInKB() { return String.valueOf(chunkLength() / 1024); }
/** * @param source Input source to read compressed data from * @param info Compression info */ public CompressedInputStream(InputStream source, CompressionInfo info, boolean hasPostCompressionAdlerChecksums) { this.info = info; this.checksum = hasPostCompressionAdlerChecksums ? new Adler32() : new CRC32(); this.hasPostCompressionAdlerChecksums = hasPostCompressionAdlerChecksums; this.buffer = new byte[info.parameters.chunkLength()]; // buffer is limited to store up to 1024 chunks this.dataBuffer = new ArrayBlockingQueue<byte[]>(Math.min(info.chunks.length, 1024)); new Thread(new Reader(source, info, dataBuffer)).start(); }
private void writeHeader(DataOutput out, long dataLength, int chunks) { try { out.writeUTF(parameters.sstableCompressor.getClass().getSimpleName()); out.writeInt(parameters.otherOptions.size()); for (Map.Entry<String, String> entry : parameters.otherOptions.entrySet()) { out.writeUTF(entry.getKey()); out.writeUTF(entry.getValue()); } // store the length of the chunk out.writeInt(parameters.chunkLength()); // store position and reserve a place for uncompressed data length and chunks count out.writeLong(dataLength); out.writeInt(chunks); } catch (IOException e) { throw new FSWriteError(e, filePath); } }
/** * @param sections Collection of sections in uncompressed file. Should not contain sections that overlap each other. * @return Total chunk size in bytes for given sections including checksum. */ public long getTotalSizeForSections(Collection<Pair<Long, Long>> sections) { long size = 0; long lastOffset = -1; for (Pair<Long, Long> section : sections) { int startIndex = (int) (section.left / parameters.chunkLength()); int endIndex = (int) (section.right / parameters.chunkLength()); endIndex = section.right % parameters.chunkLength() == 0 ? endIndex - 1 : endIndex; for (int i = startIndex; i <= endIndex; i++) { long offset = i * 8L; long chunkOffset = chunkOffsets.getLong(offset); if (chunkOffset > lastOffset) { lastOffset = chunkOffset; long nextChunkOffset = offset + 8 == chunkOffsetsSize ? compressedFileLength : chunkOffsets.getLong(offset + 8); size += (nextChunkOffset - chunkOffset); } } } return size; }
for (Pair<Long, Long> section : sections) int startIndex = (int) (section.left / parameters.chunkLength()); int endIndex = (int) (section.right / parameters.chunkLength()); endIndex = section.right % parameters.chunkLength() == 0 ? endIndex - 1 : endIndex; for (int i = startIndex; i <= endIndex; i++)
public void serialize(CompressionParameters parameters, DataOutputPlus out, int version) throws IOException { out.writeUTF(parameters.sstableCompressor.getClass().getSimpleName()); out.writeInt(parameters.otherOptions.size()); for (Map.Entry<String, String> entry : parameters.otherOptions.entrySet()) { out.writeUTF(entry.getKey()); out.writeUTF(entry.getValue()); } out.writeInt(parameters.chunkLength()); }
public long serializedSize(CompressionParameters parameters, int version) { long size = TypeSizes.NATIVE.sizeof(parameters.sstableCompressor.getClass().getSimpleName()); size += TypeSizes.NATIVE.sizeof(parameters.otherOptions.size()); for (Map.Entry<String, String> entry : parameters.otherOptions.entrySet()) { size += TypeSizes.NATIVE.sizeof(entry.getKey()); size += TypeSizes.NATIVE.sizeof(entry.getValue()); } size += TypeSizes.NATIVE.sizeof(parameters.chunkLength()); return size; } }
/** * Get a chunk of compressed data (offset, length) corresponding to given position * * @param position Position in the file. * @return pair of chunk offset and length. */ public Chunk chunkFor(long position) { // position of the chunk int idx = 8 * (int) (position / parameters.chunkLength()); if (idx >= chunkOffsetsSize) throw new CorruptSSTableException(new EOFException(), indexFilePath); long chunkOffset = chunkOffsets.getLong(idx); long nextChunkOffset = (idx + 8 == chunkOffsetsSize) ? compressedFileLength : chunkOffsets.getLong(idx + 8); return new Chunk(chunkOffset, (int) (nextChunkOffset - chunkOffset - 4)); // "4" bytes reserved for checksum }
public CompressedSequentialWriter(File file, String offsetsPath, CompressionParameters parameters, MetadataCollector sstableMetadataCollector) { super(file, parameters.chunkLength()); this.compressor = parameters.sstableCompressor; // buffer for compression should be the same size as buffer itself compressed = new ICompressor.WrappedArray(new byte[compressor.initialCompressedBufferLength(buffer.length)]); /* Index File (-CompressionInfo.db component) and it's header */ metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsPath); this.sstableMetadataCollector = sstableMetadataCollector; crcMetadata = new DataIntegrityMetadata.ChecksumWriter(out); }
offsets = this.offsets.sharedCopy(); count = (int) (dataLength / parameters.chunkLength()); if (dataLength % parameters.chunkLength() != 0) count++;