void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
public void writeTo(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.writeFixed32(blockDescriptors.size()); for (int i = 0; i < blockDescriptors.size(); i++) { blockDescriptors.get(i).writeTo(header); if (meta.hasIndexValues()) header.write(firstValues.get(i)); } header.writeTo(out); for (byte[] data : blockData) out.write(data); }
public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }
public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.<T>readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }
public static BlockDescriptor read(InputBuffer in) throws IOException { BlockDescriptor result = new BlockDescriptor(); result.rowCount = in.readFixed32(); result.uncompressedSize = in.readFixed32(); result.compressedSize = in.readFixed32(); return result; }
public void writeTo(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.writeFixed32(blockDescriptors.size()); for (int i = 0; i < blockDescriptors.size(); i++) { blockDescriptors.get(i).writeTo(header); if (meta.hasIndexValues()) header.write(firstValues.get(i)); } header.writeTo(out); for (byte[] data : blockData) out.write(data); }
private void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
public static BlockDescriptor read(InputBuffer in) throws IOException { BlockDescriptor result = new BlockDescriptor(); result.rowCount = in.readFixed32(); result.uncompressedSize = in.readFixed32(); result.compressedSize = in.readFixed32(); return result; }