public SnappyFramedInputStream(InputStream in, boolean verifyChecksums) throws IOException { this.in = in; this.verifyChecksums = verifyChecksums; allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5); this.frameHeader = new byte[4]; // stream must begin with stream header byte[] actualHeader = new byte[SnappyFramed.HEADER_BYTES.length]; int read = SnappyInternalUtils.readBytes(in, actualHeader, 0, actualHeader.length); if (read < SnappyFramed.HEADER_BYTES.length) { throw new EOFException("encountered EOF while reading stream header"); } if (!Arrays.equals(SnappyFramed.HEADER_BYTES, actualHeader)) { throw new IOException("invalid stream header"); } }
@Override public int read(byte[] output, int offset, int length) throws IOException { SnappyInternalUtils.checkNotNull(output, "output is null"); SnappyInternalUtils.checkPositionIndexes(offset, offset + length, output.length); if (closed) { throw new IOException("Stream is closed"); } if (length == 0) { return 0; } if (!ensureBuffer()) { return -1; } int size = min(length, available()); System.arraycopy(buffer, position, output, offset, size); position += size; return size; }
private boolean ensureBuffer() throws IOException if (available() > 0) { return true; if (!readBlockHeader()) { eof = true; return false; FrameMetaData frameMetaData = getFrameMetaData(frameHeader); return ensureBuffer(); allocateBuffersBasedOnSize(frameMetaData.length); FrameData frameData = getFrameData(input);
private static byte[] uncompress(byte[] compressed) throws IOException { return toByteArray(new SnappyFramedInputStream(new ByteArrayInputStream(compressed))); }
@Override public int read() throws IOException { if (closed) { return -1; } if (!ensureBuffer()) { return -1; } return buffer[position++] & 0xFF; }
private boolean ensureBuffer() throws IOException if (available() > 0) { return true; if (!readBlockHeader()) { eof = true; return false; FrameMetaData frameMetaData = getFrameMetaData(frameHeader); return ensureBuffer(); allocateBuffersBasedOnSize(frameMetaData.length); FrameData frameData = getFrameData(input);
@Test public void testInvalidChecksumIgnoredWhenVerificationDisabled() throws Exception { // flag = 0, size = 4, crc32c = 0, block data = [a] byte[] block = {1, 5, 0, 0, 0, 0, 0, 0, 'a'}; ByteArrayInputStream inputData = new ByteArrayInputStream(blockToStream(block)); assertEquals(toByteArray(new SnappyFramedInputStream(inputData, false)), new byte[] {'a'}); }
@Override public int read() throws IOException { if (closed) { return -1; } if (!ensureBuffer()) { return -1; } return buffer[position++] & 0xFF; }
@Test public void testCloseIsIdempotent() throws Exception { byte[] random = getRandom(0.5, 500000); ByteArrayOutputStream out = new ByteArrayOutputStream(); OutputStream snappyOut = new SnappyFramedOutputStream(out); snappyOut.write(random); snappyOut.close(); snappyOut.close(); byte[] compressed = out.toByteArray(); InputStream snappyIn = new SnappyFramedInputStream(new ByteArrayInputStream(compressed), true); byte[] uncompressed = toByteArray(snappyIn); assertEquals(uncompressed, random); snappyIn.close(); snappyIn.close(); }
@Override public int read(byte[] output, int offset, int length) throws IOException { SnappyInternalUtils.checkNotNull(output, "output is null"); SnappyInternalUtils.checkPositionIndexes(offset, offset + length, output.length); if (closed) { throw new IOException("Stream is closed"); } if (length == 0) { return 0; } if (!ensureBuffer()) { return -1; } int size = min(length, available()); System.arraycopy(buffer, position, output, offset, size); position += size; return size; }
public SnappyFramedInputStream(InputStream in, boolean verifyChecksums) throws IOException { this.in = in; this.verifyChecksums = verifyChecksums; allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5); this.frameHeader = new byte[4]; // stream must begin with stream header byte[] actualHeader = new byte[SnappyFramed.HEADER_BYTES.length]; int read = SnappyInternalUtils.readBytes(in, actualHeader, 0, actualHeader.length); if (read < SnappyFramed.HEADER_BYTES.length) { throw new EOFException("encountered EOF while reading stream header"); } if (!Arrays.equals(SnappyFramed.HEADER_BYTES, actualHeader)) { throw new IOException("invalid stream header"); } }
snappyInputStream, encoding)); } else if ("framing-snappy".equalsIgnoreCase(compress)) { InputStream snappyInputStream = new SnappyFramedInputStream( inputStream); reader = new BufferedReader(new InputStreamReader(
@Test public void testLargeWrites() throws Exception { byte[] random = getRandom(0.5, 500000); java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); OutputStream snappyOut = new SnappyFramedOutputStream(out); // partially fill buffer int small = 1000; snappyOut.write(random, 0, small); // write more than the buffer size snappyOut.write(random, small, random.length - small); // get compressed data snappyOut.close(); byte[] compressed = out.toByteArray(); assertTrue(compressed.length < random.length); // decompress byte[] uncompressed = uncompress(compressed); assertEquals(uncompressed, random); // decompress byte at a time InputStream in = new SnappyFramedInputStream(new ByteArrayInputStream(compressed), true); int i = 0; int c; while ((c = in.read()) != -1) { uncompressed[i++] = (byte) c; } assertEquals(i, random.length); assertEquals(uncompressed, random); }