private void startBlock(int block) throws IOException { this.block = block; this.row = column.firstRows[block]; in.seek(column.blockStarts[block]); int end = column.blocks[block].compressedSize; byte[] raw = new byte[end+checksum.size()]; in.readFully(raw); ByteBuffer data = codec.decompress(ByteBuffer.wrap(raw, 0, end)); if (!checksum.compute(data).equals (ByteBuffer.wrap(raw, end, checksum.size()))) throw new IOException("Checksums mismatch."); values = new InputBuffer(new InputBytes(data)); }
void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }
private void startBlock(int block) throws IOException { this.block = block; this.row = column.firstRows[block]; in.seek(column.blockStarts[block]); int end = column.blocks[block].compressedSize; byte[] raw = new byte[end+checksum.size()]; in.readFully(raw); ByteBuffer data = codec.decompress(ByteBuffer.wrap(raw, 0, end)); if (!checksum.compute(data).equals (ByteBuffer.wrap(raw, end, checksum.size()))) throw new IOException("Checksums mismatch."); values = new InputBuffer(new InputBytes(data)); }
private void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.<T>readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }