/** * Gets the portion of the encoded byte array from an HBase table cell that has the avro-encoded * data payload. * * @param bytes The bytes from an HBase table cell. * @return the portion of the encoded byte array that contains the binary-encoded avro message. * @throws IOException on I/O error (eg. decoding error). */ public ByteBuffer getPayload(byte[] bytes) throws IOException { final ByteStreamArray byteStream = new ByteStreamArray(bytes); // Decode the writer schema and throw it away: mSchemaDecoder.decode(byteStream); return ByteBuffer.wrap(bytes, byteStream.getOffset(), bytes.length - byteStream.getOffset()); }
/** * Serializes a long integer into bytes using the zig-zag variable-length encoding scheme. * * @param number Long integer to encode. * @return Zig-zag encoded long, as an array of up to 10 bytes. */ public static byte[] longToZigZagVarInt64(long number) { // Zig-zag encode: move sign to low-order bit, and flip others if negative number = (number << 1) ^ (number >> 63); return longToVarInt64(number); }
/** * Reads a variable-length zig-zag encoded signed integer up to 64 bits. * * @return the read integer as a long. * @throws EncodingException on decoding error. */ public long readZigZagVarInt64() throws EncodingException { final long vint64 = readVarInt64(); return (vint64 >>> 1) ^ -(vint64 & 1); // decode zig-zag back to two's-complement }
schemaId = new ByteStreamArray(result.getRow()).readVarInt64(); } catch (EncodingException exn) { LOG.error(String.format("Unable to decode schema ID encoded in row key %s: %s",
final int nbytes = sizeOfLongAsVarInt64(number); final byte[] bytes = new byte[nbytes];
/** {@inheritDoc} */ @Override public Schema decode(ByteStreamArray bstream) throws IOException { final BytesKey schemaHash = new BytesKey(bstream.readBytes(Hasher.HASH_SIZE_BYTES)); final Schema schema = mSchemaTable.getSchema(schemaHash); if (null == schema) { throw new IOException( String.format("Schema with hash %s not found in schema table.", schemaHash)); } return schema; } }
/** * Decodes the serialized bytes into a FijiCell. If reuse is non-null, the implementation may fill * it and return it as the FijiCell data payload. * * @param bytes The bytes from an HBase table cell. * @param reuse If non-null, may be filled with the decoded data and used as the data payload in * the return value. * @return The decoded FijiCell. * @throws IOException If there is an error. */ private DecodedCell<T> decode(byte[] bytes, T reuse) throws IOException { final ByteStreamArray byteStream = new ByteStreamArray(bytes); final Schema writerSchema = mSchemaDecoder.decode(byteStream); final Schema readerSchema = (mReaderSchema != null) ? mReaderSchema : writerSchema; final ByteBuffer binaryData = ByteBuffer.wrap(bytes, byteStream.getOffset(), bytes.length - byteStream.getOffset()); final T data = decodeAvro(binaryData, writerSchema, readerSchema, reuse); return new DecodedCell<T>(writerSchema, readerSchema, data); }
/** {@inheritDoc} */ @Override public void encode(final Schema writerSchema) throws IOException { final long schemaId = mCellSpec.getSchemaTable().getOrCreateSchemaId(writerSchema); mByteArrayEncoder.writeFixed(ByteStreamArray.longToVarInt64(schemaId)); } }
/** {@inheritDoc} */ @Override public Schema decode(ByteStreamArray bstream) throws IOException { final long schemaId = bstream.readVarInt64(); final Schema schema = mSchemaTable.getSchema(schemaId); if (null == schema) { throw new IOException( String.format("Schema with ID %d not found in schema table.", schemaId)); } return schema; } }
/** * Fetches a schema entry from the tables given a schema ID. * * @param schemaId schema ID * @return Avro schema entry, or null if the schema ID does not exist in the table * @throws IOException on I/O error. */ private SchemaTableEntry loadFromIdTable(long schemaId) throws IOException { final Get get = new Get(longToVarInt64(schemaId)); final Result result = mSchemaIdTable.get(get); return result.isEmpty() ? null : decodeSchemaEntry(result.value()); }
/** * Writes the given schema entry to the ID and hash tables. * * This is not protected from concurrent writes. Caller must ensure consistency. * * @param avroEntry Schema entry to write. * @param timestamp Write entries with this timestamp. * @param flush Whether to flush tables synchronously. * @throws IOException on I/O error. */ private void storeInTable(final SchemaTableEntry avroEntry, long timestamp, boolean flush) throws IOException { final byte[] entryBytes = encodeSchemaEntry(avroEntry); // Writes the ID mapping first: if the hash table write fails, we just lost one schema ID. // The hash table write must not happen before the ID table write has been persisted. // Otherwise, another client may see the hash entry, write cells with the schema ID that cannot // be decoded (since the ID mapping has not been written yet). final Put putId = new Put(longToVarInt64(avroEntry.getId())) .add(SCHEMA_COLUMN_FAMILY_BYTES, SCHEMA_COLUMN_QUALIFIER_BYTES, timestamp, entryBytes); mSchemaIdTable.put(putId); if (flush) { mSchemaIdTable.flushCommits(); } final Put putHash = new Put(avroEntry.getHash().bytes()) .add(SCHEMA_COLUMN_FAMILY_BYTES, SCHEMA_COLUMN_QUALIFIER_BYTES, timestamp, entryBytes); mSchemaHashTable.put(putHash); if (flush) { mSchemaHashTable.flushCommits(); } }