private boolean ensureBuffer() throws IOException if (available() > 0) { return true; if (!readBlockHeader()) { eof = true; return false; final FrameMetaData frameMetaData = getFrameMetaData(frameHeader); return ensureBuffer(); allocateBuffersBasedOnSize(frameMetaData.length); final FrameData frameData = getFrameData(input);
/** * @param content * @return * @throws IOException */ private FrameData getFrameData(ByteBuffer content) throws IOException { return new FrameData(getCrc32c(content), 4); }
/** * {@inheritDoc} */ @Override public int read(ByteBuffer dst) throws IOException { if (dst == null) { throw new IllegalArgumentException("dst is null"); } if (closed) { throw new ClosedChannelException(); } if (dst.remaining() == 0) { return 0; } if (!ensureBuffer()) { return -1; } final int size = min(dst.remaining(), available()); dst.put(buffer, position, size); position += size; return size; }
break; case COMPRESSION_FORMAT_SNAPPY_FRAMED: compressionIn = new SnappyFramedInputStream(bufferedIn); break; default:
@Override public int read() throws IOException { if (closed) { return -1; } if (!ensureBuffer()) { return -1; } return buffer[position++] & 0xFF; }
this.verifyChecksums = verifyChecksums; allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5); this.frameHeader = ByteBuffer.allocate(4);
@Override protected InputStream decorateWithCompression(NonClosingInputStreamDecorator stream) throws IOException { return new SnappyFramedInputStream(stream, false); } }
while (ensureBuffer()) { final int available = available(); os.write(buffer, position, available); position += available;
@Override protected InputStream decorateWithCompression(NonClosingInputStreamDecorator stream) throws IOException { return new SnappyFramedInputStream(stream, false); } }
@Override public int read(byte[] output, int offset, int length) throws IOException { if (output == null) { throw new IllegalArgumentException("output is null"); } if (offset < 0 || length < 0 || offset + length > output.length) { throw new IllegalArgumentException("invalid offset [" + offset + "] and length [" + length + ']'); } if (closed) { throw new ClosedChannelException(); } if (length == 0) { return 0; } if (!ensureBuffer()) { return -1; } final int size = min(length, available()); System.arraycopy(buffer, position, output, offset, size); position += size; return size; }
@Override protected InputStream decorateWithCompression(NonClosingInputStreamDecorator stream) throws IOException { return new SnappyFramedInputStream(stream, false); } }
while (ensureBuffer()) { bb.clear(); bb.position(position); bb.limit(position + available());
public static InputStream getInputStream(InputStream inputStream, Compression compression) throws IOException { if (compression != null) { switch (compression) { case SNAPPY: return new SnappyFramedInputStream(inputStream); case NONE: return inputStream; default: throw new IllegalArgumentException("Unsupported compression codec: " + compression); } } return inputStream; }
public static InputStream getInputStream(InputStream inputStream, Compression compression) throws IOException { if (compression != null) { switch (compression) { case SNAPPY: return new SnappyFramedInputStream(inputStream); case NONE: return inputStream; default: throw new IllegalArgumentException("Unsupported compression codec: " + compression); } } return inputStream; }
public ProtobufRecordStreamReader(TableSchema schema, List<Column> columns, InputStream in, CompressOption option) throws IOException { if (columns == null) { this.columns = schema.getColumns().toArray(new Column[0]); } else { Column[] tmpColumns = new Column[columns.size()]; for (int i = 0; i < columns.size(); ++i) { tmpColumns[i] = schema.getColumn(columns.get(i).getName()); } this.columns = tmpColumns; } bin = new BufferedInputStream(in); if (option != null) { if (option.algorithm.equals(CompressOption.CompressAlgorithm.ODPS_ZLIB)) { this.in = CodedInputStream.newInstance(new InflaterInputStream(bin)); } else if (option.algorithm.equals(CompressOption.CompressAlgorithm.ODPS_SNAPPY)) { this.in = CodedInputStream.newInstance(new SnappyFramedInputStream(bin)); } else if (option.algorithm.equals(CompressOption.CompressAlgorithm.ODPS_RAW)) { this.in = CodedInputStream.newInstance((bin)); } else { throw new IOException("invalid compression option."); } } else { this.in = CodedInputStream.newInstance(bin); } this.in.setSizeLimit(Integer.MAX_VALUE); }
break; case COMPRESSION_FORMAT_SNAPPY_FRAMED: compressionIn = new SnappyFramedInputStream(bufferedIn); break; default: