/** * Create a {@link CompressionInputStream} that will read from the given * {@link InputStream} with the given {@link Decompressor}. * * @param in the stream to read compressed bytes from * @param decompressor decompressor to use * @return a stream to read uncompressed bytes from * @throws IOException */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { checkNativeCodeLoaded(); return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); }
@Override protected int getCompressedData() throws IOException { checkStream(); // Get the size of the compressed chunk (always non-negative) int len = rawReadInt(); // Read len bytes from underlying stream if (len > buffer.length) { buffer = new byte[len]; } int n = 0, off = 0; while (n < len) { int count = in.read(buffer, off + n, len - n); if (count < 0) { throw new EOFException("Unexpected end of block in input stream"); } n += count; } return len; }
originalBlockSize = rawReadInt(); } catch (IOException ioe) { return -1; int m; try { m = getCompressedData(); } catch (EOFException e) { eof = true;
blockDecompressorStream = new BlockDecompressorStream(bytesIn, decompressor, 1024); blockDecompressorStream.read()); } catch (IOException e) { fail(joiner.join(name, e.getMessage())); try { bytesOut.close(); blockDecompressorStream.close(); bytesIn.close(); blockDecompressorStream.close(); } catch (IOException e) {
protected void getCompressedData() throws IOException { checkStream(); // Get the size of the compressed chunk int len = rawReadInt(); checkLength(len); // Read len bytes from underlying stream if (len > buffer.length) { buffer = new byte[len]; } int n = 0, off = 0; while (n < len) { int count = in.read(buffer, off + n, len - n); if (count < 0) { throw new CodecPrematureEOFException( "Unexpected end of input stream, this is typically because of" + " truncated compressed file."); } n += count; } // Send the read data to the decompressor decompressor.setInput(buffer, 0, len); }
@Test public void testRead() throws IOException { // compress empty stream bytesOut = new ByteArrayOutputStream(); BlockCompressorStream blockCompressorStream = new BlockCompressorStream(bytesOut, new FakeCompressor(), 1024, 0); // close without any write blockCompressorStream.close(); // check compressed output buf = bytesOut.toByteArray(); assertEquals("empty file compressed output size is not 4", 4, buf.length); // use compressed output as input for decompression bytesIn = new ByteArrayInputStream(buf); // get decompression stream BlockDecompressorStream blockDecompressorStream = new BlockDecompressorStream(bytesIn, new FakeDecompressor(), 1024); try { assertEquals("return value is not -1", -1 , blockDecompressorStream.read()); } catch (IOException e) { fail("unexpected IOException : " + e); } } }
public void close() throws IOException { super.close(); verifyChecksums(); } }
blockDecompressorStream = new BlockDecompressorStream(bytesIn, decompressor, 1024); blockDecompressorStream.read()); } catch (IOException e) { fail(joiner.join(name, e.getMessage())); try { bytesOut.close(); blockDecompressorStream.close(); bytesIn.close(); blockDecompressorStream.close(); } catch (IOException e) {
new BlockDecompressorStream(bytesIn, new FakeDecompressor(), 1024); try { assertEquals("return value is not -1", -1 , blockDecompressorStream.read()); } catch (IOException e) { fail("unexpected IOException : " + e); } finally { blockDecompressorStream.close();
int decompress(byte[] b, int off, int len) throws IOException { // Check if we are the beginning of a block if (noUncompressedBytes == originalBlockSize) { // Get original data size try { originalBlockSize = rawReadInt(); } catch (IOException ioe) { return -1; } noUncompressedBytes = 0; } int n = 0; while ((n = decompressor.decompress(b, off, len)) == 0) { if (decompressor.finished() || decompressor.needsDictionary()) { if (noUncompressedBytes >= originalBlockSize) { eof = true; return -1; } } if (decompressor.needsInput()) { getCompressedData(); } } // Note the no. of decompressed bytes read from 'current' block noUncompressedBytes += n; return n; }
/** * Create a {@link CompressionInputStream} that will read from the given * {@link InputStream} with the given {@link Decompressor}. * * @param in the stream to read compressed bytes from * @param decompressor decompressor to use * @return a stream to read uncompressed bytes from * @throws IOException */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { if (!isNativeCodeLoaded()) { throw new RuntimeException("native lz4 library not available"); } return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT)); }
@Override protected int getCompressedData() throws IOException { checkStream(); // Get the size of the compressed chunk (always non-negative) int len = rawReadInt(); // Read len bytes from underlying stream if (len > buffer.length) { buffer = new byte[len]; } int n = 0, off = 0; while (n < len) { int count = in.read(buffer, off + n, len - n); if (count < 0) { throw new EOFException("Unexpected end of block in input stream"); } n += count; } return len; }
new BlockDecompressorStream(bytesIn, new FakeDecompressor(), 1024); try { assertEquals("return value is not -1", -1 , blockDecompressorStream.read()); } catch (IOException e) { fail("unexpected IOException : " + e); } finally { blockDecompressorStream.close();
originalBlockSize = rawReadInt(); } catch (IOException ioe) { return -1; getCompressedData();
/** * Create a {@link CompressionInputStream} that will read from the given * {@link InputStream} with the given {@link Decompressor}. * * @param in the stream to read compressed bytes from * @param decompressor decompressor to use * @return a stream to read uncompressed bytes from * @throws IOException */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { checkNativeCodeLoaded(); return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); }
@Override protected int getCompressedData() throws IOException { checkStream(); // Get the size of the compressed chunk (always non-negative) int len = rawReadInt(); // Read len bytes from underlying stream if (len > buffer.length) { buffer = new byte[len]; } int n = 0, off = 0; while (n < len) { int count = in.read(buffer, off + n, len - n); if (count < 0) { throw new EOFException("Unexpected end of block in input stream"); } n += count; } return len; }
blockDecompressorStream = new BlockDecompressorStream(bytesIn, new Lz4Decompressor(), 1024); assertEquals("return value is not -1", -1, blockDecompressorStream.read()); } catch (Exception e) { fail("testCompressorDecompressorEmptyStreamLogic ex error !!!" bytesIn.close(); bytesOut.close(); blockDecompressorStream.close(); } catch (IOException e) {
originalBlockSize = rawReadInt(); } catch (IOException ioe) { return -1; int m; try { m = getCompressedData(); } catch (EOFException e) { eof = true;
/** * Create a {@link CompressionInputStream} that will read from the given * {@link InputStream} with the given {@link Decompressor}. * * @param in the stream to read compressed bytes from * @param decompressor decompressor to use * @return a stream to read uncompressed bytes from * @throws IOException */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { checkNativeCodeLoaded(); return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); }
@Override protected int getCompressedData() throws IOException { checkStream(); // Get the size of the compressed chunk (always non-negative) int len = rawReadInt(); // Read len bytes from underlying stream if (len > buffer.length) { buffer = new byte[len]; } int n = 0, off = 0; while (n < len) { int count = in.read(buffer, off + n, len - n); if (count < 0) { throw new EOFException("Unexpected end of block in input stream"); } n += count; } return len; }