@Test public void testLockAcquisitionAndRelease() throws Exception { addTableInput(); QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); }
@Override public void close() throws IOException { try { LOG.info("DriverCleanup for LLAP splits: {}", applicationId); driver.releaseLocksAndCommitOrRollback(true); driver.close(); driver.destroy(); txnManager.closeTxnManager(); } catch (Exception err) { LOG.error("Error closing driver resources", err); throw new IOException(err); } }
switch(work.getOperationType()) { case REPL_OPEN_TXN: List<Long> txnIds = txnManager.replOpenTxn(replPolicy, work.getTxnIds(), user); assert txnIds.size() == work.getTxnIds().size(); LOG.info("Replayed OpenTxn Event for policy " + replPolicy + " with srcTxn " + if (txnManager.isTxnOpen()) { long txnId = txnManager.getCurrentTxnId(); txnManager.commitTxn(); LOG.info("Committed txn from REPL_MIGRATION_OPEN_TXN : " + txnId); Long txnIdMigration = txnManager.openTxn(driverContext.getCtx(), user); long writeId = txnManager.getTableWriteId(work.getDbName(), work.getTableName()); String validTxnList = txnManager.getValidTxns().toString(); conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList); conf.set(ReplUtils.REPL_CURRENT_TBL_WRITE_ID, Long.toString(writeId)); case REPL_ABORT_TXN: for (long txnId : work.getTxnIds()) { txnManager.replRollbackTxn(replPolicy, txnId); LOG.info("Replayed AbortTxn Event for policy " + replPolicy + " with srcTxn " + txnId); case REPL_MIGRATION_COMMIT_TXN: assert (work.getReplLastIdInfo() != null); long txnIdMigrationCommit = txnManager.getCurrentTxnId(); CommitTxnRequest commitTxnRequestMigr = new CommitTxnRequest(txnIdMigrationCommit); commitTxnRequestMigr.setReplLastIdInfo(work.getReplLastIdInfo()); txnManager.replCommitTxn(commitTxnRequestMigr); conf.unset(ValidTxnList.VALID_TXNS_KEY);
private void recordValidTxns(HiveTxnManager txnMgr) throws LockException { String oldTxnString = conf.get(ValidTxnList.VALID_TXNS_KEY); if ((oldTxnString != null) && (oldTxnString.length() > 0)) { throw new IllegalStateException("calling recordValidTxn() more than once in the same " + JavaUtils.txnIdToString(txnMgr.getCurrentTxnId())); } ValidTxnList txnList = txnMgr.getValidTxns(); String txnStr = txnList.toString(); conf.set(ValidTxnList.VALID_TXNS_KEY, txnStr); LOG.debug("Encoding valid txns info " + txnStr + " txnid:" + txnMgr.getCurrentTxnId()); }
private void openTransaction() throws LockException, CommandProcessorResponse { if (checkConcurrency() && startImplicitTxn(queryTxnMgr)) { String userFromUGI = getUserFromUGI(); if (!queryTxnMgr.isTxnOpen()) { if (userFromUGI == null) { throw createProcessorResponse(10); } queryTxnMgr.openTxn(ctx, userFromUGI); } } }
/** * Initialize the transaction manager. This is done lazily to avoid hard wiring one * transaction manager at the beginning of the session. * @param conf Hive configuration to initialize transaction manager * @return transaction manager * @throws LockException */ public synchronized HiveTxnManager initTxnMgr(HiveConf conf) throws LockException { // Only change txnMgr if the setting has changed if (txnMgr != null && !txnMgr.getTxnManagerName().equals(conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER))) { txnMgr.closeTxnManager(); txnMgr = null; } if (txnMgr == null) { txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } return txnMgr; }
swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("update tab1 set b = 7 where b=1", true)); long idTxnUpdate1 = txnMgr2.getCurrentTxnId(); txnMgr2.acquireLocks(driver.getPlan(), ctx, "T2"); List<ShowLocksResponseElement> locks = getLocks(txnMgr2); Assert.assertEquals("Unexpected lock count", 2, locks.size()); long idTxnUpdate2 = txnMgr.getCurrentTxnId(); ((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T3", false); locks = getLocks(txnMgr); long writeId = txnMgr2.getTableWriteId("default", "tab1"); AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), writeId, "default", "tab1", Collections.singletonList("p=one")); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); txnMgr2.commitTxn();//txnid:idTxnUpdate1 ((DbLockManager)txnMgr.getLockManager()).checkLock(locks.get(2).getLockid());//retest WAITING locks (both have same ext id) locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks); writeId = txnMgr.getTableWriteId("default", "tab1"); adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), writeId, "default", "tab1", Collections.singletonList("p=two")); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn();//txnid:idTxnUpdate2
"partitioned by (p int, q int) clustered by (a) into 2 buckets " + "stored as orc TBLPROPERTIES ('transactional'='true')")); long txnid1 = txnMgr.openTxn(ctx, "T1"); checkCmdOnDriver(driver.compileAndRespond("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1)); txnMgr.rollbackTxn(); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + long txnid2 = txnMgr.openTxn(ctx, "T1"); checkCmdOnDriver(driver.compileAndRespond("insert into target partition(p=1,q) values (10,2,2), (30,4,2), (50,6,3), (70,8,2)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); long writeId = txnMgr.getTableWriteId("default", "target"); AddDynamicPartitions adp = new AddDynamicPartitions(txnid2, writeId, "default", "target", Arrays.asList("p=1/q=2","p=1/q=2")); adp.setOperationType(DataOperationType.INSERT); 2,//2 distinct partitions modified TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); txnMgr.commitTxn();
@Test public void testRollback() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE); QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.rollbackTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); }
@Test public void createTable() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.compileAndRespond("create table if not exists T (a int, b int)", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); } @Test
if(!queryTxnMgr.isTxnOpen() && queryTxnMgr.supportsAcid()) { for (FileSinkDesc desc : acidSinks) { TableDesc tableInfo = desc.getTableInfo(); long writeId = queryTxnMgr.getTableWriteId(Utilities.getDatabaseName(tableInfo.getTableName()), Utilities.getTableName(tableInfo.getTableName())); desc.setTableWriteId(writeId); desc.setStatementId(queryTxnMgr.getStmtIdAndIncrement()); String unionAllSubdir = "/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX; if(desc.getInsertOverwrite() && desc.getDirName().toString().contains(unionAllSubdir) && queryTxnMgr.getTableWriteId(t.getDbName(), t.getTableName()); if (hasAcidDdl) { String fqTableName = acidDdlDesc.getFullTableName(); long writeId = queryTxnMgr.getTableWriteId( Utilities.getDatabaseName(fqTableName), Utilities.getTableName(fqTableName)); acidDdlDesc.setWriteId(writeId); queryTxnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState); if (queryTxnMgr.recordSnapshot(plan) && !validTxnListsGenerated) { throw new IllegalStateException( "Need to record valid WriteID list but there is no valid TxnID list (" + JavaUtils.txnIdToString(queryTxnMgr.getCurrentTxnId()) + ", queryId:" + plan.getQueryId() + ")");
HiveTxnManager txnMgr = ss.getTxnMgr(); if(startTxnImplicitly) { assert !txnMgr.getAutoCommit(); if((txnMgr.getAutoCommit() && haveAcidWrite()) || plan.getOperation() == HiveOperation.START_TRANSACTION || (!txnMgr.getAutoCommit() && startTxnImplicitly)) { if(txnMgr.isTxnOpen()) { throw new RuntimeException("Already have an open transaction txnid:" + txnMgr.getCurrentTxnId()); txnMgr.openTxn(ctx, userFromUGI); initiatingTransaction = true; readOnlyQueryInAutoCommit = txnMgr.getAutoCommit() && plan.getOperation() == HiveOperation.QUERY && !haveAcidWrite(); desc.setTransactionId(txnMgr.getCurrentTxnId()); desc.setStatementId(txnMgr.getWriteIdAndIncrement()); txnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState); if(initiatingTransaction || (readOnlyQueryInAutoCommit && acidInQuery)) {
long baseTxnId = txnMgr.openTxn(ctx, "u0"); long baseWriteId = txnMgr.getTableWriteId("temp", "T7"); Assert.assertEquals(1, baseWriteId); txnMgr.commitTxn(); // committed baseTxnId long underHwmOpenTxnId = txnMgr1.openTxn(ctx, "u1"); Assert.assertTrue("Invalid txn ID", underHwmOpenTxnId > baseTxnId); long testTxnId = txnMgr2.openTxn(ctx, "u2"); Assert.assertTrue("Invalid txn ID", testTxnId > underHwmOpenTxnId); String testValidTxns = txnMgr2.getValidTxns().toString(); ValidWriteIdList testValidWriteIds = txnMgr2.getValidWriteIds(Collections.singletonList("temp.t7"), testValidTxns) .getTableValidWriteIdList("temp.t7"); Assert.assertEquals(baseWriteId, testValidWriteIds.getHighWatermark()); long aboveHwmOpenTxnId = txnMgr3.openTxn(ctx, "u3"); Assert.assertTrue("Invalid txn ID", aboveHwmOpenTxnId > testTxnId); long aboveHwmOpenWriteId = txnMgr3.getTableWriteId("temp", "T7"); Assert.assertEquals(2, aboveHwmOpenWriteId); long underHwmOpenWriteId = txnMgr1.getTableWriteId("temp", "T7"); Assert.assertEquals(3, underHwmOpenWriteId); testValidWriteIds = txnMgr2.getValidWriteIds(Collections.singletonList("temp.t7"), testValidTxns) .getTableValidWriteIdList("temp.t7"); Assert.assertEquals(underHwmOpenWriteId, testValidWriteIds.getHighWatermark()); txnMgr1.commitTxn(); testValidWriteIds = txnMgr2.getValidWriteIds(Collections.singletonList("temp.t7"), testValidTxns)
/** * txns overlap in time but do not update same resource - no conflict */ @Test public void testWriteSetTracking2() throws Exception { dropTable(new String[] {"TAB_PART", "TAB2"}); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr.openTxn(ctx, "Peter"); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Peter"); txnMgr2.openTxn(ctx, "Catherine"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); //note that "update" uses dynamic partitioning thus lock is on the table not partition checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB_PART", null, locks); txnMgr.commitTxn(); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 9 where p = 'doh'", true)); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Catherine"); txnMgr2.commitTxn(); } /**
return; if (txnMgr.isTxnOpen()) { if (commit) { if(conf.getBoolVar(ConfVars.HIVE_IN_TEST) && conf.getBoolVar(ConfVars.HIVETESTMODEROLLBACKTXN)) { txnMgr.rollbackTxn(); txnMgr.commitTxn();//both commit & rollback clear ALL locks for this tx txnMgr.rollbackTxn(); hiveLocks.addAll(ctx.getHiveLocks()); txnMgr.releaseLocks(hiveLocks);
cpr = driver.compileAndRespond("select a from T7 ", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets S lock on T7 HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T7", "p=1", locks); txnMgr.commitTxn();//release locks from "select a from T7" - to unblock hte drop partition lockState = ((DbLockManager)txnMgr2.getLockManager()).checkLock(locks.get(6).getLockid()); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 4, locks.size()); checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=2", locks); txnMgr2.rollbackTxn();//release the X lock on T7.p=1 lockState = ((DbLockManager)txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid());//S lock on T7 locks = getLocks(); Assert.assertEquals("Unexpected lock count", 3, locks.size());
txnMgr.openTxn(ctx, "Long Running"); checkCmdOnDriver(driver.compileAndRespond("select a from TAB_PART where p = 'blah'", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); txnMgr2.openTxn(ctx, "Short Running"); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'blah'", true));//no such partition txnMgr2.acquireLocks(driver.getPlan(), ctx, "Short Running"); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); rqst.setTxnIds(Collections.singletonList(txnMgr2.getCurrentTxnId())); AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(rqst); Assert.assertEquals(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId()); AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.EMPTY_LIST); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); txnMgr2.commitTxn(); txnMgr2.openTxn(ctx, "T3"); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'", true));//pretend this partition exists txnMgr2.acquireLocks(driver.getPlan(), ctx, "T3"); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); rqst.setTxnIds(Collections.singletonList(txnMgr2.getCurrentTxnId()));
@Test public void testDDLNoLock() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_NO_LOCK); QueryPlan qp = new MockQueryPlan(this, HiveOperation.CREATEDATABASE); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertNull(locks); txnMgr.rollbackTxn(); }
@Test public void lockConflictDbTable() throws Exception { dropTable(new String[] {"temp.T7"}); CommandProcessorResponse cpr = driver.run("create database if not exists temp"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists temp.T7(a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("update temp.T7 set a = 5 where b = 6", true);//gets SS lock on T7 checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("drop database if exists temp", true)); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "temp", "T7", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "temp", null, null, locks); txnMgr.commitTxn(); ((DbLockManager)txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid()); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "temp", null, null, locks); txnMgr2.commitTxn(); } @Test
cpr = driver.compileAndRespond("select a from T6", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets S lock on T6 List<HiveLock> selectLocks = ctx.getHiveLocks(); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks); txnMgr.rollbackTxn();//release S on T6 lockState = ((DbLockManager)txnMgr.getLockManager()).checkLock(locks.get(1).getLockid()); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T6", null, locks); txnMgr2.rollbackTxn(); cpr = driver.run("drop table if exists T6"); locks = getLocks();