private static Map<String, String> updateDbProps(Database dbObj, String dumpDirectory) { /* explicitly remove the setting of last.repl.id from the db object parameters as loadTask is going to run multiple times and explicit logic is in place which prevents updates to tables when db level last repl id is set and we create a AlterDatabaseTask at the end of processing a database. */ Map<String, String> parameters = new HashMap<>(dbObj.getParameters()); parameters.remove(ReplicationSpec.KEY.CURR_STATE_ID.toString()); // Add the checkpoint key to the Database binding it to current dump directory. // So, if retry using same dump, we shall skip Database object update. parameters.put(ReplUtils.REPL_CHECKPOINT_KEY, dumpDirectory); return parameters; }
public static String getReplPolicyIdString(Database db) { if (db != null) { Map<String, String> m = db.getParameters(); if ((m != null) && (m.containsKey(SOURCE_OF_REPLICATION))) { String replPolicyId = m.get(SOURCE_OF_REPLICATION); LOG.debug("repl policy for database {} is {}", db.getName(), replPolicyId); return replPolicyId; } LOG.debug("Repl policy is not set for database ", db.getName()); } return null; }
private boolean isDbAlreadyBootstrapped(Database db) { Map<String, String> props = db.getParameters(); return ((props != null) && props.containsKey(ReplicationSpec.KEY.CURR_STATE_ID.toString()) && !props.get(ReplicationSpec.KEY.CURR_STATE_ID.toString()).isEmpty()); }
public static boolean isBootstrapDumpInProgress(Hive hiveDb, String dbName) throws HiveException { Database database = hiveDb.getDatabase(dbName); if (database == null) { return false; } Map<String, String> params = database.getParameters(); if (params == null) { return false; } for (String key : params.keySet()) { if (key.startsWith(BOOTSTRAP_DUMP_STATE_KEY_PREFIX) && params.get(key).equals(ReplDumpState.ACTIVE.name())) { return true; } } return false; }
public static void resetDbBootstrapDumpState(Hive hiveDb, String dbName, String uniqueKey) throws HiveException { Database database = hiveDb.getDatabase(dbName); if (database != null) { Map<String, String> params = database.getParameters(); if ((params != null) && params.containsKey(uniqueKey)) { params.remove(uniqueKey); database.setParameters(params); hiveDb.alterDatabase(dbName, database); LOG.info("REPL DUMP:: Reset property for Database: {}, Property: {}", dbName, uniqueKey); } } }
private int showCreateDatabase(Hive db, DataOutputStream outStream, String databaseName) throws Exception { Database database = db.getDatabase(databaseName); StringBuilder createDb_str = new StringBuilder(); createDb_str.append("CREATE DATABASE `").append(database.getName()).append("`\n"); if (database.getDescription() != null) { createDb_str.append("COMMENT\n '"); createDb_str.append( HiveStringUtils.escapeHiveCommand(database.getDescription())).append("'\n"); } createDb_str.append("LOCATION\n '"); createDb_str.append(database.getLocationUri()).append("'\n"); String propertiesToString = propertiesToString(database.getParameters(), null); if (!propertiesToString.isEmpty()) { createDb_str.append("WITH DBPROPERTIES (\n"); createDb_str.append(propertiesToString).append(")\n"); } outStream.write(createDb_str.toString().getBytes("UTF-8")); return 0; }
public static void createDbExportDump(FileSystem fs, Path metadataPath, Database dbObj, ReplicationSpec replicationSpec) throws IOException, SemanticException { // WARNING NOTE : at this point, createDbExportDump lives only in a world where ReplicationSpec is in replication scope // If we later make this work for non-repl cases, analysis of this logic might become necessary. Also, this is using // Replv2 semantics, i.e. with listFiles laziness (no copy at export time) // Remove all the entries from the parameters which are added by repl tasks internally. Map<String, String> parameters = dbObj.getParameters(); if (parameters != null) { Map<String, String> tmpParameters = new HashMap<>(parameters); tmpParameters.entrySet() .removeIf(e -> e.getKey().startsWith(Utils.BOOTSTRAP_DUMP_STATE_KEY_PREFIX) || e.getKey().equals(ReplUtils.REPL_CHECKPOINT_KEY) || e.getKey().equals(ReplChangeManager.SOURCE_OF_REPLICATION)); dbObj.setParameters(tmpParameters); } try (JsonWriter jsonWriter = new JsonWriter(fs, metadataPath)) { new DBSerializer(dbObj).writeTo(jsonWriter, replicationSpec); } if (parameters != null) { dbObj.setParameters(parameters); } }
HCatDatabase(Database db) { this.dbName = db.getName(); this.props = db.getParameters(); this.dbLocation = db.getLocationUri(); this.comment = db.getDescription(); }
private void createEndReplLogTask(Context context, Scope scope, ReplLogger replLogger) throws SemanticException { Database dbInMetadata = work.databaseEvent(context.hiveConf).dbInMetadata(work.dbNameToLoadIn); ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger, dbInMetadata.getParameters()); Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork); if (scope.rootTasks.isEmpty()) { scope.rootTasks.add(replLogTask); } else { DAGTraversal.traverse(scope.rootTasks, new AddDependencyToLeaves(Collections.singletonList(replLogTask))); } }
@Override public TaskTracker tasks() throws SemanticException { Database dbObj = readDbMetadata(); tracker.addTask(alterDbTask(dbObj.getName(), dbObj.getParameters(), context.hiveConf)); return tracker; } }
public static String setDbBootstrapDumpState(Hive hiveDb, String dbName) throws HiveException { Database database = hiveDb.getDatabase(dbName); if (database == null) { return null; } Map<String, String> newParams = new HashMap<>(); String uniqueKey = BOOTSTRAP_DUMP_STATE_KEY_PREFIX + UUID.randomUUID().toString(); newParams.put(uniqueKey, ReplDumpState.ACTIVE.name()); Map<String, String> params = database.getParameters(); // if both old params are not null, merge them if (params != null) { params.putAll(newParams); database.setParameters(params); } else { // if one of them is null, replace the old params with the new one database.setParameters(newParams); } hiveDb.alterDatabase(dbName, database); LOG.info("REPL DUMP:: Set property for Database: {}, Property: {}, Value: {}", dbName, uniqueKey, Utils.ReplDumpState.ACTIVE.name()); return uniqueKey; }
private TaskTracker forNewTable() throws Exception { Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); // If table doesn't exist, allow creating a new one only if the database state is older than the update. // This in-turn applicable for partitions creation as well. if ((parentDb != null) && (!event.replicationSpec().allowReplacementInto(parentDb.getParameters()))) { return tracker; } Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator(); while (iterator.hasNext() && tracker.canAddMoreTasks()) { AddPartitionDesc currentPartitionDesc = iterator.next(); /* the currentPartitionDesc cannot be inlined as we need the hasNext() to be evaluated post the current retrieved lastReplicatedPartition */ addPartition(iterator.hasNext(), currentPartitionDesc, null); } return tracker; }
private boolean shouldReplayEvent(FileStatus dir, DumpType dumpType, String dbName, String tableName) { // if database itself is null then we can not filter out anything. if (dbName == null || dbName.isEmpty()) { return true; } else if ((tableName == null) || (tableName.isEmpty())) { Database database; try { database = Hive.get().getDatabase(dbName); return database == null ? true : isEventNotReplayed(database.getParameters(), dir, dumpType); } catch (HiveException e) { //may be the db is getting created in this load log.debug("failed to get the database " + dbName); return true; } } else { Table tbl; try { tbl = Hive.get().getTable(dbName, tableName); return isEventNotReplayed(tbl.getParameters(), dir, dumpType); } catch (HiveException e) { // may be the table is getting created in this load log.debug("failed to get the table " + dbName + "." + tableName); return true; } } }
public static Database fromMetastoreApiDatabase(org.apache.hadoop.hive.metastore.api.Database database) { String ownerName = "PUBLIC"; PrincipalType ownerType = PrincipalType.ROLE; if (database.getOwnerName() != null) { ownerName = database.getOwnerName(); ownerType = fromMetastoreApiPrincipalType(database.getOwnerType()); } Map<String, String> parameters = database.getParameters(); if (parameters == null) { parameters = ImmutableMap.of(); } return Database.builder() .setDatabaseName(database.getName()) .setLocation(Optional.ofNullable(database.getLocationUri())) .setOwnerName(ownerName) .setOwnerType(ownerType) .setComment(Optional.ofNullable(database.getDescription())) .setParameters(parameters) .build(); }
private ReplLoadOpType getLoadDbType(String dbName) throws InvalidOperationException, HiveException { Database db = context.hiveDb.getDatabase(dbName); if (db == null) { return ReplLoadOpType.LOAD_NEW; } if (isDbAlreadyBootstrapped(db)) { throw new InvalidOperationException("Bootstrap REPL LOAD is not allowed on Database: " + dbName + " as it was already done."); } if (ReplUtils.replCkptStatus(dbName, db.getParameters(), context.dumpDirectory)) { return ReplLoadOpType.LOAD_SKIP; } if (isDbEmpty(dbName)) { return ReplLoadOpType.LOAD_REPLACE; } throw new InvalidOperationException("Bootstrap REPL LOAD is not allowed on Database: " + dbName + " as it is not empty. One or more tables/functions exist."); }
public Object getFieldValue(_Fields field) { switch (field) { case NAME: return getName(); case DESCRIPTION: return getDescription(); case LOCATION_URI: return getLocationUri(); case PARAMETERS: return getParameters(); case PRIVILEGES: return getPrivileges(); case OWNER_NAME: return getOwnerName(); case OWNER_TYPE: return getOwnerType(); case CATALOG_NAME: return getCatalogName(); case CREATE_TIME: return getCreateTime(); } throw new IllegalStateException(); }
@Override public void createDatabase(Database db) throws InvalidObjectException, MetaException { boolean commited = false; MDatabase mdb = new MDatabase(); assert db.getCatalogName() != null; mdb.setCatalogName(normalizeIdentifier(db.getCatalogName())); assert mdb.getCatalogName() != null; mdb.setName(db.getName().toLowerCase()); mdb.setLocationUri(db.getLocationUri()); mdb.setDescription(db.getDescription()); mdb.setParameters(db.getParameters()); mdb.setOwnerName(db.getOwnerName()); PrincipalType ownerType = db.getOwnerType(); mdb.setOwnerType((null == ownerType ? PrincipalType.USER.name() : ownerType.name())); mdb.setCreateTime(db.getCreateTime()); try { openTransaction(); pm.makePersistent(mdb); commited = commitTransaction(); } finally { if (!commited) { rollbackTransaction(); } } }
@Test public void testDefaultDatabaseData() throws Exception { Database database = client.getDatabase(DEFAULT_DATABASE); Assert.assertEquals("Default database name", "default", database.getName()); Assert.assertEquals("Default database description", "Default Hive database", database.getDescription()); Assert.assertEquals("Default database location", metaStore.getWarehouseRoot(), new Path(database.getLocationUri())); Assert.assertEquals("Default database parameters", new HashMap<String, String>(), database.getParameters()); Assert.assertEquals("Default database owner", "public", database.getOwnerName()); Assert.assertEquals("Default database owner type", PrincipalType.ROLE, database.getOwnerType()); Assert.assertNull("Default database privileges", database.getPrivileges()); Assert.assertTrue("database create time should be set", database.isSetCreateTime()); Assert.assertTrue("Database create time should be non-zero", database.getCreateTime() > 0); }
@Test public void testCreateDatabaseDefaultValues() throws Exception { Database database = new DatabaseBuilder() .setName("dummy") .create(client, metaStore.getConf()); Database createdDatabase = client.getDatabase(database.getName()); Assert.assertNull("Comparing description", createdDatabase.getDescription()); Assert.assertEquals("Comparing location", metaStore.getWarehouseRoot() + "/" + createdDatabase.getName() + ".db", createdDatabase.getLocationUri()); Assert.assertEquals("Comparing parameters", new HashMap<String, String>(), createdDatabase.getParameters()); Assert.assertNull("Comparing privileges", createdDatabase.getPrivileges()); Assert.assertEquals("Comparing owner name", SecurityUtils.getUser(), createdDatabase.getOwnerName()); Assert.assertEquals("Comparing owner type", PrincipalType.USER, createdDatabase.getOwnerType()); }
@Test public void testAlterDatabaseNotNullableFields() throws Exception { Database database = getDatabaseWithAllParametersSet(); client.createDatabase(database); Database originalDatabase = client.getDatabase(database.getName()); Database newDatabase = new Database(); newDatabase.setName("new_name"); newDatabase.setCatalogName(DEFAULT_CATALOG_NAME); client.alterDatabase(originalDatabase.getName(), newDatabase); // The name should not be changed, so reload the db with the original name Database alteredDatabase = client.getDatabase(originalDatabase.getName()); Assert.assertEquals("Database name should not change", originalDatabase.getName(), alteredDatabase.getName()); Assert.assertEquals("Database description should not change", originalDatabase.getDescription(), alteredDatabase.getDescription()); Assert.assertEquals("Database location should not change", originalDatabase.getLocationUri(), alteredDatabase.getLocationUri()); Assert.assertEquals("Database parameters should be empty", new HashMap<String, String>(), alteredDatabase.getParameters()); Assert.assertNull("Database owner should be empty", alteredDatabase.getOwnerName()); Assert.assertEquals("Database owner type should not change", originalDatabase.getOwnerType(), alteredDatabase.getOwnerType()); Assert.assertNull("Database privileges should be empty", alteredDatabase.getPrivileges()); }