/** * Initialize the transaction manager. This is done lazily to avoid hard wiring one * transaction manager at the beginning of the session. * @param conf Hive configuration to initialize transaction manager * @return transaction manager * @throws LockException */ public synchronized HiveTxnManager initTxnMgr(HiveConf conf) throws LockException { // Only change txnMgr if the setting has changed if (txnMgr != null && !txnMgr.getTxnManagerName().equals(conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER))) { txnMgr.closeTxnManager(); txnMgr = null; } if (txnMgr == null) { txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } return txnMgr; }
/** * Get the singleton instance of this factory. * @return this factory */ public static synchronized TxnManagerFactory getTxnManagerFactory() { if (self == null) { self = new TxnManagerFactory(); } return self; }
/** * Initialize the transaction manager. This is done lazily to avoid hard wiring one * transaction manager at the beginning of the session. * @param conf Hive configuration to initialize transaction manager * @return transaction manager * @throws LockException */ public synchronized HiveTxnManager initTxnMgr(HiveConf conf) throws LockException { // Only change txnMgr if the setting has changed if (txnMgr != null && !txnMgr.getTxnManagerName().equals(conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER))) { txnMgr.closeTxnManager(); txnMgr = null; } if (txnMgr == null) { txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } return txnMgr; }
/** * Get the singleton instance of this factory. * @return this factory */ public static synchronized TxnManagerFactory getTxnManagerFactory() { if (self == null) { self = new TxnManagerFactory(); } return self; }
@Override public void start(HiveConf hiveConf) throws Exception { this.hiveConf = hiveConf; HiveTxnManager mgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(hiveConf); if(!mgr.supportsAcid()) { LOG.info(this.getClass().getName() + " not started since " + mgr.getClass().getName() + " does not support Acid."); return;//there are no transactions in this case } pool = Executors.newScheduledThreadPool(1, new ThreadFactory() { private final AtomicInteger threadCounter = new AtomicInteger(); @Override public Thread newThread(Runnable r) { return new Thread(r, HouseKeeperServiceBase.this.getClass().getName() + "-" + threadCounter.getAndIncrement()); } }); TimeUnit tu = TimeUnit.MILLISECONDS; pool.scheduleAtFixedRate(getScheduedAction(hiveConf, isAliveCounter), getStartDelayMs(), getIntervalMs(), tu); LOG.info("Started " + this.getClass().getName() + " with delay/interval = " + getStartDelayMs() + "/" + getIntervalMs() + " " + tu); }
/** * Get the singleton instance of this factory. * @return this factory */ public static synchronized TxnManagerFactory getTxnManagerFactory() { if (self == null) { self = new TxnManagerFactory(); } return self; }
@Before public void setUp() throws Exception { conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true); conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, DummyTxnManager.class.getName()); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); SessionState.start(conf); ctx = new Context(conf); txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); Assert.assertTrue(txnMgr instanceof DummyTxnManager); // Use reflection to set LockManager since creating the object using the // relection in DummyTxnManager won't take Mocked object Field field = DummyTxnManager.class.getDeclaredField("lockMgr"); field.setAccessible(true); field.set(txnMgr, mockLockManager); Field field2 = DummyTxnManager.class.getDeclaredField("lockManagerCtx"); field2.setAccessible(true); field2.set(txnMgr, mockLockManagerCtx); }
@Before public void setUp() throws Exception { TxnDbUtil.prepDb(conf); txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr.getLockManager();//init lock manager Assert.assertTrue(txnMgr instanceof DbTxnManager); nextInput = 1; readEntities = new HashSet<ReadEntity>(); writeEntities = new HashSet<WriteEntity>(); conf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS); conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 10, TimeUnit.SECONDS); houseKeeperService = new AcidHouseKeeperService(); houseKeeperService.setConf(conf); }
@Test public void concurrencyFalse() throws Exception { HiveConf badConf = new HiveConf(); if(badConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY)) { //TxnManagerFactory is a singleton, so if the default is true, it has already been //created and won't throw return; } badConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); badConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); boolean sawException = false; try { TxnManagerFactory.getTxnManagerFactory().getTxnManager(badConf); } catch (RuntimeException e) { sawException = true; } Assert.assertTrue(sawException); }
/** * Add the task according to the parsed command tree. This is used for the CLI * command "SHOW LOCKS DATABASE database [extended];". * * @param ast * The parsed command tree. * @throws SemanticException * Parsing failed */ private void analyzeShowDbLocks(ASTNode ast) throws SemanticException { boolean isExtended = (ast.getChildCount() > 1); String dbName = stripQuotes(ast.getChild(0).getText()); HiveTxnManager txnManager = null; try { txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } catch (LockException e) { throw new SemanticException(e.getMessage()); } ShowLocksDesc showLocksDesc = new ShowLocksDesc(ctx.getResFile(), dbName, isExtended, txnManager.useNewShowLocksFormat()); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), showLocksDesc))); setFetchTask(createFetchTask(showLocksDesc.getSchema())); // Need to initialize the lock manager ctx.setNeedLockMgr(true); }
/** * Add the task according to the parsed command tree. This is used for the CLI * command "SHOW LOCKS DATABASE database [extended];". * * @param ast * The parsed command tree. * @throws SemanticException * Parsing failed */ private void analyzeShowDbLocks(ASTNode ast) throws SemanticException { boolean isExtended = (ast.getChildCount() > 1); String dbName = stripQuotes(ast.getChild(0).getText()); HiveTxnManager txnManager = null; try { txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } catch (LockException e) { throw new SemanticException(e.getMessage()); } ShowLocksDesc showLocksDesc = new ShowLocksDesc(ctx.getResFile(), dbName, isExtended, txnManager.useNewShowLocksFormat()); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), showLocksDesc), conf)); setFetchTask(createFetchTask(showLocksDesc.getSchema())); // Need to initialize the lock manager ctx.setNeedLockMgr(true); }
txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } catch (LockException e) { throw new SemanticException(e.getMessage());
checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets S lock on T6 HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table if exists T6", true); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks); HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr3);
txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } catch (LockException e) { throw new SemanticException(e.getMessage());
/** * check that locks in Waiting state show what they are waiting on * This test is somewhat abusive in that it make DbLockManager retain locks for 2 * different queries (which are not part of the same transaction) which can never * happen in real use cases... but it makes testing convenient. * @throws Exception */ @Test public void testLockBlockedBy() throws Exception { dropTable(new String[] {"TAB_BLOCKED"}); CommandProcessorResponse cpr = driver.run("create table TAB_BLOCKED (a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select * from TAB_BLOCKED", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "I AM SAM"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table TAB_BLOCKED", true); checkCmdOnDriver(cpr); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "SAM I AM", false);//make non-blocking locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "TAB_BLOCKED", null, locks); Assert.assertEquals("BlockedByExtId doesn't match", locks.get(0).getLockid(), locks.get(1).getBlockedByExtId()); Assert.assertEquals("BlockedByIntId doesn't match", locks.get(0).getLockIdInternal(), locks.get(1).getBlockedByIntId()); }
driver2.setMaxRows(10000); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(hiveConf); HiveTxnManager txnMgr1 = swapTxnManager(txnMgr2); Driver driver1 = swapDrivers(driver2);
/** * txns update same resource but do not overlap in time - no conflict */ @Test public void testWriteSetTracking1() throws Exception { dropTable(new String[] {"TAB_PART"}); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.compileAndRespond("select * from TAB_PART", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Nicholas"); txnMgr.commitTxn(); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Alexandra"); txnMgr2.commitTxn(); } private void dropTable(String[] tabs) throws Exception {
HiveTxnManager txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); Driver driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null, null, txnManager); DriverCleanup driverCleanup = new DriverCleanup(driver, txnManager, splitsAppId.toString());
@Test public void lockConflictDbTable() throws Exception { dropTable(new String[] {"temp.T7"}); CommandProcessorResponse cpr = driver.run("create database if not exists temp"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists temp.T7(a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("update temp.T7 set a = 5 where b = 6", true);//gets SS lock on T7 checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("drop database if exists temp", true)); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "temp", "T7", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "temp", null, null, locks); txnMgr.commitTxn(); ((DbLockManager)txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid()); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "temp", null, null, locks); txnMgr2.commitTxn(); } @Test
HiveTxnManager txnMgr1 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr1); cpr = driver.compileAndRespond("insert into table db1.t14 partition (ds='today') values (1, 2)", true); txnMgr1.acquireLocks(driver.getPlan(), ctx, "Tom"); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("insert into table db1.t14 partition (ds='tomorrow') values (3, 4)", true); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Jerry"); HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr3); cpr = driver.compileAndRespond("select * from db2.t15", true); txnMgr3.acquireLocks(driver.getPlan(), ctx, "Donald"); HiveTxnManager txnMgr4 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr4); cpr = driver.compileAndRespond("select * from db2.t16", true); txnMgr4.acquireLocks(driver.getPlan(), ctx, "Hillary"); HiveTxnManager txnMgr5 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr5); cpr = driver.compileAndRespond("select * from db2.t14", true);