public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }
public <T extends Comparable> T readValue(ValueType type) throws IOException { switch (type) { case NULL: return (T)null; case BOOLEAN: return (T)Boolean.valueOf(readBoolean()); case INT: return (T)Integer.valueOf(readInt()); case LONG: return (T)Long.valueOf(readLong()); case FIXED32: return (T)Integer.valueOf(readFixed32()); case FIXED64: return (T)Long.valueOf(readFixed64()); case FLOAT: return (T)Float.valueOf(readFloat()); case DOUBLE: return (T)Double.valueOf(readDouble()); case STRING: return (T)readString(); case BYTES: return (T)readBytes(null); default: throw new TrevniRuntimeException("Unknown value type: "+type); } }
/** Seek to the named row. */ public void seek(long r) throws IOException { if (r < row || r >= column.lastRow(block)) // not in current block startBlock(column.findBlock(r)); // seek to block start while (r > row && hasNext()) { // skip within block values.skipValue(type); row++; } previous = null; }
static void read(InputBuffer in, MetaData<?> metaData) throws IOException { int size = in.readInt(); for (int i = 0; i < size; i++) metaData.put(in.readString(), in.readBytes()); }
@Test public void testSkipBytes() throws Exception { long sentinel = Long.MAX_VALUE; OutputBuffer out = new OutputBuffer(); out.writeValue("trevni".getBytes(), ValueType.BYTES); out.writeLong(sentinel); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.BYTES); Assert.assertEquals(sentinel, in.readLong()); } @Test public void testInitPos() throws Exception {
@Test public void testInitPos() throws Exception { long sentinel = Long.MAX_VALUE; OutputBuffer out = new OutputBuffer(); out.writeValue(Integer.MAX_VALUE, ValueType.INT); out.writeLong(sentinel); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.readInt(); long pos = in.tell(); in = new InputBuffer(new InputBytes(out.toByteArray()), pos); Assert.assertEquals(sentinel, in.readLong()); } }
public void skipValue(ValueType type) throws IOException { switch (type) { case NULL: break; case BOOLEAN: readBoolean(); break; case INT: readInt(); break; case LONG: readLong(); break; case FIXED32: case FLOAT: skip(4); break; case FIXED64: case DOUBLE: skip(8); break; case STRING: case BYTES: skipBytes(); break; default: throw new TrevniRuntimeException("Unknown value type: "+type); } }
private void readHeader() throws IOException { InputBuffer in = new InputBuffer(file, 0); readMagic(in); this.rowCount = in.readFixed64(); this.columnCount = in.readFixed32(); this.metaData = ColumnFileMetaData.read(in); this.columnsByName = new HashMap<>(columnCount); columns = new ColumnDescriptor[columnCount]; readColumnMetaData(in); readColumnStarts(in); }
@Test public void testInt() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeInt(random.nextInt()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextInt(), in.readInt()); }
@Test public void testEmpty() throws Exception { OutputBuffer out = new OutputBuffer(); ByteArrayOutputStream temp = new ByteArrayOutputStream(); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); Assert.assertEquals(0, in.tell()); Assert.assertEquals(0, in.length()); }
@Test public void testLong() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeLong(random.nextLong()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextLong(), in.readLong()); }
@Test public void testString() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeString(TestUtil.randomString(random)); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(TestUtil.randomString(random), in.readString()); } @Test public void testSkipNull() throws Exception {
@Test public void testFixed32() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFixed32(random.nextInt()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextInt(), in.readFixed32()); }
@Test public void testFloat() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFloat(random.nextFloat()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextFloat(), in.readFloat(), 0); }
@Test public void testBytes() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeBytes(TestUtil.randomBytes(random)); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(TestUtil.randomBytes(random), in.readBytes(null)); }
@Test public void testFixed64() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFixed64(random.nextLong()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextLong(), in.readFixed64()); }
@Test public void testDouble() throws Exception { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeDouble(Double.MIN_VALUE); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); for (int i = 0; i < COUNT; i++) Assert.assertEquals(Double.MIN_VALUE, in.readDouble(), 0); }
private void startBlock(int block) throws IOException { this.block = block; this.row = column.firstRows[block]; in.seek(column.blockStarts[block]); int end = column.blocks[block].compressedSize; byte[] raw = new byte[end+checksum.size()]; in.readFully(raw); ByteBuffer data = codec.decompress(ByteBuffer.wrap(raw, 0, end)); if (!checksum.compute(data).equals (ByteBuffer.wrap(raw, end, checksum.size()))) throw new IOException("Checksums mismatch."); values = new InputBuffer(new InputBytes(data)); }