/** * Stores the specified schema entry in memory. * * External synchronization required. * * @param avroEntry Avro schema entry. * @return the SchemaEntry stored in memory. */ private SchemaEntry storeInMemory(final SchemaTableEntry avroEntry) { return storeInMemory(fromAvroEntry(avroEntry)); }
/** {@inheritDoc} */ @Override public BytesKey getOrCreateSchemaHash(final Schema schema) throws IOException { return getOrCreateSchemaEntry(schema).getHash(); }
/** * Looks up a schema entry given an Avro schema object. * * Looks first in-memory. If the schema is not known in-memory, looks in the HTables. * * @param schema Avro schema to look up. * @return Either the pre-existing entry for the specified schema, or a newly created entry. * @throws IOException on I/O error. */ private synchronized SchemaEntry getOrCreateSchemaEntry(final Schema schema) throws IOException { final State state = mState.get(); Preconditions.checkState(state == State.OPEN, "Cannot get or create schema entry from SchemaTable instance in state %s.", state); final BytesKey schemaHash = getSchemaHash(schema); final SchemaEntry knownEntry = getSchemaEntry(schemaHash); if (knownEntry != null) { return knownEntry; } // Schema is unknown, both in-memory and in-table. // Allocate a new schema ID and write it down to the tables: return storeInMemory(registerNewSchemaInTable(schema, schemaHash)); }
/** {@inheritDoc} */ @Override public SchemaEntry getSchemaEntry(Schema schema) throws IOException { return getSchemaEntry(getSchemaHash(schema)); }
final Set<SchemaEntry> hashTableEntries = loadSchemaHashTable(mSchemaHashTable); final Set<SchemaEntry> idTableEntries = loadSchemaIdTable(mSchemaIdTable); if (!checkConsistency(mergedEntries)) { LOG.error("Merged schema hash and ID tables are inconsistent"); new HashSet<SchemaEntry>(avroBackupEntries.size()); for (SchemaTableEntry avroEntry : avroBackupEntries) { schemaTableEntries.add(fromAvroEntry(avroEntry)); if (!checkConsistency(schemaTableEntries)) { LOG.error("Backup schema entries are inconsistent"); if (!checkConsistency(schemaTableEntries)) { LOG.error("Backup schema entries are inconsistent with already existing schema entries"); flush(); SchemaPlatformBridge.get().setWriteBufferSize(mSchemaIdTable, schemaTableEntries.size() + 1); SchemaPlatformBridge.get().setWriteBufferSize(mSchemaHashTable, schemaTableEntries.size()); storeInTable(toAvroEntry(entry), timestamp, false); // do not flush setSchemaIdCounter(nextSchemaId); flush(); } finally { mZKLock.unlock();
try { final Set<SchemaEntry> hashTableEntries = loadSchemaHashTable(mSchemaHashTable); if (!checkConsistency(hashTableEntries)) { LOG.error("Schema hash table is inconsistent"); final Set<SchemaEntry> idTableEntries = loadSchemaIdTable(mSchemaIdTable); if (!checkConsistency(idTableEntries)) { LOG.error("Schema hash table is inconsistent"); if (!checkConsistency(mergedEntries)) { LOG.error("Merged schema hash and ID tables are inconsistent"); entries.add(toAvroEntry(entry));
mZKLock.lock(); try { final SchemaTableEntry existingAvroEntry = loadFromHashTable(schemaHash); if (existingAvroEntry != null) { return fromAvroEntry(existingAvroEntry); storeInTable(toAvroEntry(entry)); return entry;
SCHEMA_COLUMN_QUALIFIER_BYTES)) { try { final SchemaEntry entry = fromAvroEntry(decodeSchemaEntry(keyValue.getValue())); entries.add(entry); if (!getSchemaHash(entry.getSchema()).equals(entry.getHash())) { LOG.error(String.format( "Invalid schema hash table entry: computed schema hash %s does not match entry %s", getSchemaHash(entry.getSchema()), entry));
admin.createTable(idTableDescriptor); final HBaseSchemaTable schemaTable = new HBaseSchemaTable(fijiURI, conf, tableFactory); try { schemaTable.setSchemaIdCounter(0L); schemaTable.registerPrimitiveSchemas(); } finally { ResourceUtils.closeOrLog(schemaTable);
/** {@inheritDoc} */ @Override public Schema getSchema(BytesKey schemaHash) throws IOException { final SchemaEntry entry = getSchemaEntry(schemaHash); return (entry == null) ? null : entry.getSchema(); }
/** * Fetches a schema entry from the tables given a schema hash. * * @param schemaHash schema hash * @return Avro schema entry, or null if the schema hash does not exist in the table * @throws IOException on I/O error. */ private SchemaTableEntry loadFromHashTable(BytesKey schemaHash) throws IOException { final Get get = new Get(schemaHash.getBytes()); final Result result = mSchemaHashTable.get(get); return result.isEmpty() ? null : decodeSchemaEntry(result.value()); }
/** * Pre-registers all the primitive data types. * * @throws IOException on I/O failure. */ private synchronized void registerPrimitiveSchemas() throws IOException { int expectedSchemaId = 0; LOG.debug("Pre-registering primitive schema types."); for (PreRegisteredSchema desc : PreRegisteredSchema.values()) { final Schema schema = Schema.create(desc.getType()); Preconditions.checkState(getOrCreateSchemaId(schema) == expectedSchemaId); Preconditions.checkState(desc.getSchemaId() == expectedSchemaId); expectedSchemaId += 1; } Preconditions.checkState(expectedSchemaId == PRE_REGISTERED_SCHEMA_COUNT); }
/** * Disables and removes the schema table from HBase. * * @param admin The HBase Admin object. * @param fijiURI The FijiURI for the instance to remove. * @throws IOException If there is an error. */ public static void uninstall(HBaseAdmin admin, FijiURI fijiURI) throws IOException { final String hashTableName = FijiManagedHBaseTableName.getSchemaHashTableName(fijiURI.getInstance()).toString(); deleteTable(admin, hashTableName); final String idTableName = FijiManagedHBaseTableName.getSchemaIdTableName(fijiURI.getInstance()).toString(); deleteTable(admin, idTableName); }
/** {@inheritDoc} */ @Override public synchronized void close() throws IOException { flush(); final State oldState = mState.getAndSet(State.CLOSED); Preconditions.checkState(oldState == State.OPEN, "Cannot close SchemaTable instance in state %s.", oldState); ResourceTracker.get().unregisterResource(this); ResourceUtils.closeOrLog(mSchemaHashTable); ResourceUtils.closeOrLog(mSchemaIdTable); ResourceUtils.closeOrLog(mZKLock); ResourceUtils.closeOrLog(mZKClient); }
/** * Writes the given schema entry to the ID and hash tables. * * This is not protected from concurrent writes. Caller must ensure consistency. * * @param avroEntry Schema entry to write. * @param timestamp Write entries with this timestamp. * @param flush Whether to flush tables synchronously. * @throws IOException on I/O error. */ private void storeInTable(final SchemaTableEntry avroEntry, long timestamp, boolean flush) throws IOException { final byte[] entryBytes = encodeSchemaEntry(avroEntry); // Writes the ID mapping first: if the hash table write fails, we just lost one schema ID. // The hash table write must not happen before the ID table write has been persisted. // Otherwise, another client may see the hash entry, write cells with the schema ID that cannot // be decoded (since the ID mapping has not been written yet). final Put putId = new Put(longToVarInt64(avroEntry.getId())) .add(SCHEMA_COLUMN_FAMILY_BYTES, SCHEMA_COLUMN_QUALIFIER_BYTES, timestamp, entryBytes); mSchemaIdTable.put(putId); if (flush) { mSchemaIdTable.flushCommits(); } final Put putHash = new Put(avroEntry.getHash().bytes()) .add(SCHEMA_COLUMN_FAMILY_BYTES, SCHEMA_COLUMN_QUALIFIER_BYTES, timestamp, entryBytes); mSchemaHashTable.put(putHash); if (flush) { mSchemaHashTable.flushCommits(); } }
throw kie; mSchemaTable = new HBaseSchemaTable(mURI, mConf, mHTableFactory); mMetaTable = new HBaseMetaTable(mURI, mConf, mSchemaTable, mHTableFactory);
SCHEMA_COLUMN_QUALIFIER_BYTES)) { try { final SchemaEntry entry = fromAvroEntry(decodeSchemaEntry(keyValue.getValue())); entries.add(entry); if (!getSchemaHash(entry.getSchema()).equals(entry.getHash())) { LOG.error(String.format("Invalid schema hash table entry with row key %s: " + "computed schema hash %s does not match entry %s", rowKey, getSchemaHash(entry.getSchema()), entry));
/** {@inheritDoc} */ @Override public synchronized Schema getSchema(long schemaId) throws IOException { final SchemaEntry entry = getSchemaEntry(schemaId); return (entry == null) ? null : entry.getSchema(); }
/** * Fetches a schema entry from the tables given a schema ID. * * @param schemaId schema ID * @return Avro schema entry, or null if the schema ID does not exist in the table * @throws IOException on I/O error. */ private SchemaTableEntry loadFromIdTable(long schemaId) throws IOException { final Get get = new Get(longToVarInt64(schemaId)); final Result result = mSchemaIdTable.get(get); return result.isEmpty() ? null : decodeSchemaEntry(result.value()); }
/** {@inheritDoc} */ @Override public long getOrCreateSchemaId(final Schema schema) throws IOException { return getOrCreateSchemaEntry(schema).getId(); }