@Override ByteBuffer compress(ByteBuffer in) throws IOException { ByteBuffer out = ByteBuffer.allocate(Snappy.maxCompressedLength(in.remaining())); int size = Snappy.compress(in.array(), in.position(), in.remaining(), out.array(), 0); out.limit(size); return out; }
@Override public DataPoint getDataPoint(long timestamp, KDataInput buffer) throws IOException { int buffSz = buffer.readUnsignedShort(); byte[] byteBuffer = new byte[buffSz]; buffer.readFully(byteBuffer, 0, buffSz); String result; if (Snappy.isValidCompressedBuffer(byteBuffer, 0, buffSz)) { byte[] uncompressedArray = new byte[Snappy.uncompressedLength(byteBuffer, 0, buffSz)]; int incompressedLength = Snappy.uncompress(byteBuffer, 0, buffSz, uncompressedArray, 0); result = new String(uncompressedArray, 0, incompressedLength, UTF8); } else { result = new String(byteBuffer, UTF8); } SnappyStringDataPoint ret = new SnappyStringDataPoint(timestamp, result); return ret; }
/** * High-level API for uncompressing the input byte array. * * @param input * @return the uncompressed byte array * @throws IOException */ public static byte[] uncompress(byte[] input) throws IOException { byte[] result = new byte[Snappy.uncompressedLength(input)]; Snappy.uncompress(input, 0, input.length, result, 0); return result; }
@Override public int compress(ByteBuffer inDecompressed, ByteBuffer outCompressed) throws IOException { return Snappy.compress(inDecompressed, outCompressed); } }
public static byte[] decompressWithSnappy(byte[] bytes) { try { if (!Snappy.isValidCompressedBuffer(bytes)) { throw new IllegalArgumentException("Cannot decompress these bytes using Snappy"); } return Snappy.uncompress(bytes); } catch (IOException e) { throw Throwables.throwUncheckedException(e); } } }
@Override public int decompress(ByteBuffer compressedInput, ByteBuffer decompressedOutput) throws IOException { return Snappy.uncompress(compressedInput, decompressedOutput); } }
private boolean hasSufficientOutputBufferFor(int inputSize) { int maxCompressedSize = Snappy.maxCompressedLength(inputSize); return maxCompressedSize < outputBuffer.length - outputCursor - 4; }
/** * Uncompress the input as a double array * * @param input * @return the uncompressed data * @throws IOException */ public static double[] uncompressDoubleArray(byte[] input) throws IOException { int uncompressedLength = Snappy.uncompressedLength(input, 0, input.length); double[] result = new double[uncompressedLength / 8]; impl.rawUncompress(input, 0, input.length, result, 0); return result; }
private SnappyCompressor() { // this would throw java.lang.NoClassDefFoundError if Snappy class // wasn't found at runtime which should be processed by the calling method Snappy.getNativeLibraryVersion(); }
@Override protected void decode(ChannelHandlerContext ctx, FrameCodec.Frame msg, List<Object> out) throws Exception { byte[] in = new byte[msg.size]; msg.payload.read(in); long uncompressedLength = Snappy.uncompressedLength(in) & 0xFFFFFFFFL; if (uncompressedLength > MAX_SIZE) { logger.info("{}: uncompressed frame size exceeds the limit ({} bytes), drop the peer", channel, uncompressedLength); channel.disconnect(ReasonCode.BAD_PROTOCOL); return; } byte[] uncompressed = new byte[(int) uncompressedLength]; try { Snappy.rawUncompress(in, 0, in.length, uncompressed, 0); } catch (IOException e) { String detailMessage = e.getMessage(); // 5 - error code for framed snappy if (detailMessage.startsWith("FAILED_TO_UNCOMPRESS") && detailMessage.contains("5")) { logger.info("{}: Snappy frames are not allowed in DEVp2p protocol, drop the peer", channel); channel.disconnect(ReasonCode.BAD_PROTOCOL); return; } else { throw e; } } out.add(new FrameCodec.Frame((int) msg.type, uncompressed)); }
/** * Returns true iff the contents of compressed buffer [offset, * offset+length) can be uncompressed successfully. Does not return the * uncompressed data. Takes time proportional to the input length, but is * usually at least a factor of four faster than actual decompression. */ public static boolean isValidCompressedBuffer(byte[] input) throws IOException { return isValidCompressedBuffer(input, 0, input.length); }
/** * Uncompress the input[offset, offset+length) as a String of the given * encoding * * @param input * @param offset * @param length * @param encoding * @return the uncompressed data * @throws IOException */ public static String uncompressString(byte[] input, int offset, int length, Charset encoding) throws IOException, UnsupportedEncodingException { byte[] uncompressed = new byte[uncompressedLength(input, offset, length)]; uncompress(input, offset, length, uncompressed, 0); return new String(uncompressed, encoding); }
public static byte[] compress(byte[] input) throws IOException { if(!useCompression) return input; return Snappy.compress(input); }
public static byte[] decompressWithSnappy(byte[] bytes) { try { if (!Snappy.isValidCompressedBuffer(bytes)) { throw new IllegalArgumentException("Cannot decompress these bytes using Snappy"); } return Snappy.uncompress(bytes); } catch (IOException e) { throw Throwables.throwUncheckedException(e); } } }
public static byte[] decompress(byte[] input) throws IOException { if(!useCompression) return input; return Snappy.uncompress(input); }
/** * Compress the input data and produce a byte array of the uncompressed data * * @param data input array. The input MUST be an array type * @param byteSize the input byte size * @return compressed data */ public static byte[] rawCompress(Object data, int byteSize) throws IOException { byte[] buf = new byte[Snappy.maxCompressedLength(byteSize)]; int compressedByteSize = impl.rawCompress(data, 0, byteSize, buf, 0); byte[] result = new byte[compressedByteSize]; System.arraycopy(buf, 0, result, 0, compressedByteSize); return result; }
/** * Uncompress the input[offset, .., offset+length) as a char array * * @param input * @param offset * @param length * @return the uncompressed data * @throws IOException */ public static char[] uncompressCharArray(byte[] input, int offset, int length) throws IOException { int uncompressedLength = Snappy.uncompressedLength(input, offset, length); char[] result = new char[uncompressedLength / 2]; impl.rawUncompress(input, offset, length, result, 0); return result; }
public static SnappyCompressor create(Map<String, String> compressionOptions) { // this would throw java.lang.NoClassDefFoundError if Snappy class // wasn't found at runtime which should be processed by calling method Snappy.getNativeLibraryVersion(); // no specific options supported so far return instance; }
private ByteBuf decompressDirect(ByteBuf input) throws IOException { ByteBuffer in = inputNioBuffer(input); // Increase reader index. input.readerIndex(input.writerIndex()); if (!Snappy.isValidCompressedBuffer(in)) throw new DriverInternalError("Provided frame does not appear to be Snappy compressed"); // If the input is direct we will allocate a direct output buffer as well as this will allow us // to use // Snappy.compress(ByteBuffer, ByteBuffer) and so eliminate memory copies. ByteBuf output = input.alloc().directBuffer(Snappy.uncompressedLength(in)); try { ByteBuffer out = outputNioBuffer(output); int size = Snappy.uncompress(in, out); // Set the writer index so the amount of written bytes is reflected output.writerIndex(output.writerIndex() + size); } catch (IOException e) { // release output buffer so we not leak and rethrow exception. output.release(); throw e; } return output; }
@Override public ByteBuffer compress(ByteBuffer in) throws IOException { ByteBuffer out = ByteBuffer.allocate(Snappy.maxCompressedLength(in.remaining())+4); int size = Snappy.compress(in.array(), in.position(), in.remaining(), out.array(), 0); crc32.reset(); crc32.update(in.array(), in.position(), in.remaining()); out.putInt(size, (int)crc32.getValue()); out.limit(size+4); return out; }