private void writeHeader(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.write(MAGIC); // magic header.writeFixed64(rowCount); // row count header.writeFixed32(columnCount); // column count metaData.write(header); // file metadata for (ColumnOutputBuffer column : columns) column.getMeta().write(header); // column metadata for (long start : computeStarts(header.size())) header.writeFixed64(start); // column starts header.writeTo(out); }
public void writeValue(Object value) throws IOException { buffer.writeValue(value, meta.getType()); if (meta.hasIndexValues() && rowCount == 0) firstValues.add(buffer.toByteArray()); }
break; case BOOLEAN: writeBoolean((Boolean)value); break; case INT: writeInt((Integer)value); break; case LONG: writeLong((Long)value); break; case FIXED32: writeFixed32((Integer)value); break; case FIXED64: writeFixed64((Long)value); break; case FLOAT: writeFloat((Float)value); break; case DOUBLE: writeDouble((Double)value); break; case STRING: writeString((String)value); break; case BYTES: if (value instanceof ByteBuffer) writeBytes((ByteBuffer)value); else writeBytes((byte[])value); break; default:
@Test public void testSkipBytes() throws Exception { long sentinel = Long.MAX_VALUE; OutputBuffer out = new OutputBuffer(); out.writeValue("trevni".getBytes(), ValueType.BYTES); out.writeLong(sentinel); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); in.skipValue(ValueType.BYTES); Assert.assertEquals(sentinel, in.readLong()); } @Test public void testInitPos() throws Exception {
@Test public void testZero() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); out.writeInt(0); byte[] bytes = out.toByteArray(); Assert.assertEquals(1, bytes.length); Assert.assertEquals(0, bytes[0]); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); Assert.assertEquals(0, in.readInt()); }
@Test public void testLong() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeLong(random.nextLong()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextLong(), in.readLong()); }
public void writeTo(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.writeFixed32(blockDescriptors.size()); for (int i = 0; i < blockDescriptors.size(); i++) { blockDescriptors.get(i).writeTo(header); if (meta.hasIndexValues()) header.write(firstValues.get(i)); } header.writeTo(out); for (byte[] data : blockData) out.write(data); }
void write(OutputBuffer out) throws IOException { out.writeInt(size()); for (Map.Entry<String,byte[]> e : entrySet()) { out.writeString(e.getKey()); out.writeBytes(e.getValue()); } }
@Test public void testString() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeString(TestUtil.randomString(random)); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(TestUtil.randomString(random), in.readString()); } @Test public void testSkipNull() throws Exception {
@Test public void testFixed32() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFixed32(random.nextInt()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextInt(), in.readFixed32()); }
@Test public void testBytes() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeBytes(TestUtil.randomBytes(random)); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(TestUtil.randomBytes(random), in.readBytes(null)); }
@Test public void testFloat() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFloat(random.nextFloat()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextFloat(), in.readFloat(), 0); }
@Test public void testFixed64() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeFixed64(random.nextLong()); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextLong(), in.readFixed64()); }
@Test public void testDouble() throws Exception { OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeDouble(Double.MIN_VALUE); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); for (int i = 0; i < COUNT; i++) Assert.assertEquals(Double.MIN_VALUE, in.readDouble(), 0); }
@Test public void testBoolean() throws Exception { Random random = TestUtil.createRandom(); OutputBuffer out = new OutputBuffer(); for (int i = 0; i < COUNT; i++) out.writeValue(random.nextBoolean(), ValueType.BOOLEAN); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) Assert.assertEquals(random.nextBoolean(), in.readValue(ValueType.BOOLEAN)); }
@Test public void testEmpty() throws Exception { OutputBuffer out = new OutputBuffer(); ByteArrayOutputStream temp = new ByteArrayOutputStream(); InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray())); Assert.assertEquals(0, in.tell()); Assert.assertEquals(0, in.length()); }
void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
public void writeBytes(byte[] bytes) { writeBytes(bytes, 0, bytes.length); }
public void writeTo(OutputBuffer out) throws IOException { out.writeFixed32(rowCount); out.writeFixed32(uncompressedSize); out.writeFixed32(compressedSize); }