/** * Hashes an Avro schema. * * @param schema Avro schema to hash. * @return the schema hash. */ public BytesKey getHash(Schema schema) { final BytesKey hash = mCache.get(schema); if (null != hash) { return hash; } final BytesKey newHash = new BytesKey(hashSchema(schema)); mCache.put(schema, newHash); return newHash; } }
/** {@inheritDoc} */ @Override public boolean equals(Object other) { if (!(other instanceof SchemaEntry)) { return false; } final SchemaEntry entry = (SchemaEntry) other; return (this.mId == entry.mId) && (this.mHash.equals(entry.mHash)) && (this.mSchema.equals(entry.mSchema)); }
/** {@inheritDoc} */ @Override public void encode(final Schema writerSchema) throws IOException { final BytesKey schemaHash = mCellSpec.getSchemaTable().getOrCreateSchemaHash(writerSchema); mByteArrayEncoder.writeFixed(schemaHash.getBytes()); } }
LOG.error(String.format( "Invalid schema hash table row key size: %s, expecting %d bytes.", new BytesKey(result.getRow()), Hasher.HASH_SIZE_BYTES)); continue; final BytesKey rowKey = new BytesKey(result.getRow()); for (KeyValue keyValue : result.getColumn(SCHEMA_COLUMN_FAMILY_BYTES, SCHEMA_COLUMN_QUALIFIER_BYTES)) { final SchemaEntry entry = fromAvroEntry(decodeSchemaEntry(keyValue.getValue())); entries.add(entry); if (!getSchemaHash(entry.getSchema()).equals(entry.getHash())) { LOG.error(String.format( "Invalid schema hash table entry: computed schema hash %s does not match entry %s", getSchemaHash(entry.getSchema()), entry)); if (!rowKey.equals(entry.getHash())) { LOG.error(String.format("Inconsistent schema hash table: " + "hash encoded in row key %s does not match schema entry: %s",
/** * Register a schema. * * @return Tool exit code. * @throws IOException in case of an error. */ private int registerSchema() throws IOException { final FijiSchemaTable table = mFiji.getSchemaTable(); final File file = new File(mRegisterFlag); final Schema schema = new Schema.Parser().parse(file); final long id = table.getOrCreateSchemaId(schema); final String hash = table.getSchemaHash(schema).toString(); if (isInteractive()) { getPrintStream().print("Schema ID for the given schema is: "); } getPrintStream().println(id); if (isInteractive()) { getPrintStream().print("Schema hash for the given schema is: "); } getPrintStream().println(hash); return SUCCESS; }
final BytesKey rowKey = new BytesKey(result.getRow()); final SchemaEntry entry = fromAvroEntry(decodeSchemaEntry(keyValue.getValue())); entries.add(entry); if (!getSchemaHash(entry.getSchema()).equals(entry.getHash())) { LOG.error(String.format("Invalid schema hash table entry with row key %s: " + "computed schema hash %s does not match entry %s",
final BytesKey rowKey = new BytesKey( ByteUtils.toBytes(row.getBytesUnsafe(SCHEMA_COLUMN_ID_KEY))); final SchemaEntry entry = fromAvroEntry(decodeSchemaEntry(schemaAsBytes)); entries.add(entry); if (!getSchemaHash(entry.getSchema()).equals(entry.getHash())) { LOG.error("Invalid schema hash table entry with row key {}: " + "computed schema hash {} does not match entry {}.",
/** * Fetches a schema entry from the tables given a schema hash. * * @param schemaHash schema hash * @return Avro schema entry, or null if the schema hash does not exist in the table * @throws IOException on I/O error. */ private SchemaTableEntry loadFromHashTable(BytesKey schemaHash) throws IOException { final Get get = new Get(schemaHash.getBytes()); final Result result = mSchemaHashTable.get(get); return result.isEmpty() ? null : decodeSchemaEntry(result.value()); }
/** {@inheritDoc} */ @Override public Schema decode(ByteStreamArray bstream) throws IOException { final BytesKey schemaHash = new BytesKey(bstream.readBytes(Hasher.HASH_SIZE_BYTES)); final Schema schema = mSchemaTable.getSchema(schemaHash); if (null == schema) { throw new IOException( String.format("Schema with hash %s not found in schema table.", schemaHash)); } return schema; } }
/** {@inheritDoc} */ @Override public synchronized SchemaEntry getSchemaEntry(BytesKey schemaHash) throws IOException { final State state = mState.get(); Preconditions.checkState(state == State.OPEN, "Cannot get schema entry from SchemaTable instance in state %s.", state); final SchemaEntry existingEntry = mSchemaHashMap.get(schemaHash); if (existingEntry != null) { return existingEntry; } // On a lookup miss from the local schema cache, check to see if we can get the schema // from the original HBase table, cache it locally, and return it. final SchemaTableEntry avroEntry = loadFromHashTable(schemaHash); if (null == avroEntry) { return null; } final SchemaEntry entry = storeInMemory(avroEntry); Preconditions.checkState(schemaHash.equals(entry.getHash())); return entry; }
final BytesKey rowKey = new BytesKey(ByteUtils.toBytes(hashKey)); final long timestamp = row.getDate(SCHEMA_COLUMN_TIME).getTime(); final byte[] schemaAsBytes = ByteUtils.toBytes(row.getBytes(SCHEMA_COLUMN_VALUE)); final SchemaEntry entry = fromAvroEntry(decodeSchemaEntry(schemaAsBytes)); entries.add(entry); if (!getSchemaHash(entry.getSchema()).equals(entry.getHash())) { LOG.error( "Invalid schema hash table entry: computed schema hash {} does not match entry {}.", getSchemaHash(entry.getSchema()), entry); if (!rowKey.equals(entry.getHash())) { LOG.error( "Inconsistent schema hash table: hash encoded in row key {}"
/** * Fetches a schema entry from the tables given a schema hash. * * @param schemaHash schema hash * @return Avro schema entry, or null if the schema hash does not exist in the table * @throws IOException on I/O error. */ private SchemaTableEntry loadFromHashTable(BytesKey schemaHash) throws IOException { final ByteBuffer tableKey = ByteBuffer.wrap(schemaHash.getBytes()); final ResultSet resultSet = mAdmin.execute(mPreparedStatementReadHashTable.bind(tableKey)); final List<Row> rows = resultSet.all(); if (0 == rows.size()) { return null; } assert(rows.size() == 1); final byte[] schemaAsBytes = ByteUtils.toBytes(rows.get(0).getBytes(SCHEMA_COLUMN_VALUE)); return decodeSchemaEntry(schemaAsBytes); }
/** * Converts an Avro SchemaTableEntry into a SchemaEntry. * * @param avroEntry Avro SchemaTableEntry * @return an equivalent SchemaEntry */ public static SchemaEntry fromAvroEntry(final SchemaTableEntry avroEntry) { final String schemaJson = avroEntry.getAvroSchema(); final Schema schema = new Schema.Parser().parse(schemaJson); return new SchemaEntry(avroEntry.getId(), new BytesKey(avroEntry.getHash().bytes()), schema); }
/** {@inheritDoc} */ @Override public SchemaEntry getSchemaEntry(BytesKey schemaHash) throws IOException { final State state = mState.get(); Preconditions.checkState(state == State.OPEN, "Cannot get schema entry from SchemaTable instance in state %s.", state); SchemaEntry existingEntry = mSchemaHashMap.get(schemaHash); if (existingEntry != null) { return existingEntry; } synchronized (this) { existingEntry = mSchemaHashMap.get(schemaHash); if (existingEntry != null) { return existingEntry; } // On a lookup miss from the local schema cache, check to see if we can get the schema // from the schema hash table, cache it locally, and return it. final SchemaTableEntry avroEntry = loadFromHashTable(schemaHash); if (null == avroEntry) { return null; } final SchemaEntry entry = storeInMemory(fromAvroEntry(avroEntry)); Preconditions.checkState(schemaHash.equals(entry.getHash())); return entry; } }
/** * Converts a SchemaEntry into an Avro SchemaTableEntry. * * @param entry a SchemaEntry. * @return an equivalent Avro SchemaTableEntry. */ public static SchemaTableEntry toAvroEntry(final SchemaEntry entry) { return SchemaTableEntry.newBuilder().setId(entry.getId()) .setHash(new MD5Hash(entry.getHash().getBytes())) .setAvroSchema(entry.getSchema().toString()).build(); }
/** * Converts an Avro SchemaTableEntry into a SchemaEntry. * * @param avroEntry Avro SchemaTableEntry * @return an equivalent SchemaEntry */ public static SchemaEntry fromAvroEntry(final SchemaTableEntry avroEntry) { final String schemaJson = avroEntry.getAvroSchema(); final Schema schema = new Schema.Parser().parse(schemaJson); return new SchemaEntry(avroEntry.getId(), new BytesKey(avroEntry.getHash().bytes()), schema); }
if (existingEntryWithHash.getHash().equals(entry.getHash()) && existingEntryWithHash.getSchema().equals(entry.getSchema())) {
/** * Converts a SchemaEntry into an Avro SchemaTableEntry. * * @param entry a SchemaEntry. * @return an equivalent Avro SchemaTableEntry. */ public static SchemaTableEntry toAvroEntry(final SchemaEntry entry) { return SchemaTableEntry .newBuilder() .setId(entry.getId()) .setHash(new MD5Hash(entry.getHash().getBytes())) .setAvroSchema(entry.getSchema().toString()) .build(); }
final BytesKey bytesKey = new BytesKey(ByteArrayFormatter.parseHex(mGetByHashFlag, ':')); final SchemaEntry sEntry = table.getSchemaEntry(bytesKey); final Schema schema = sEntry.getSchema();
if (existingEntryWithHash.getHash().equals(entry.getHash()) && existingEntryWithHash.getSchema().equals(entry.getSchema())) {