void setUpInternalExtended(boolean isOrcFormat) throws Exception { hiveConf.setBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING, true); hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict"); hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "none"); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); hiveConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); runStatementOnDriver("create table " + TableExtended.NONACIDPART + "(a int, b int) partitioned by (p string) stored as orc TBLPROPERTIES ('transactional'='false')"); if (!isOrcFormat) { runStatementOnDriver("create table " + TableExtended.MMTBL + "(a int, b int) TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); runStatementOnDriver("create table " + TableExtended.MMTBL2 + "(a int, b int) TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); runStatementOnDriver("create table " + TableExtended.MMTBLPART + "(a int, b int) partitioned by (p string) TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); } else { runStatementOnDriver("create table " + TableExtended.MMTBL + "(a int, b int) stored as orc TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); runStatementOnDriver("create table " + TableExtended.MMTBL2 + "(a int, b int) stored as orc TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); runStatementOnDriver("create table " + TableExtended.MMTBLPART + "(a int, b int) partitioned by (p string) stored as orc TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); } } protected void dropTables() throws Exception {
protected void dropTables() throws Exception { super.dropTables(); for(TestTxnCommandsForMmTable.TableExtended t : TestTxnCommandsForMmTable.TableExtended.values()) { runStatementOnDriver("drop table if exists " + t); } } /**
private void verifyDirAndResult(int expectedDeltas) throws Exception { FileSystem fs = FileSystem.get(hiveConf); // Verify the content of subdirs FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (TableExtended.MMTBL).toString().toLowerCase()), FileUtils.HIDDEN_FILES_PATH_FILTER); int sawDeltaTimes = 0; for (int i = 0; i < status.length; i++) { Assert.assertTrue(status[i].getPath().getName().matches("delta_.*")); sawDeltaTimes++; FileStatus[] files = fs.listStatus(status[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); Assert.assertEquals(1, files.length); Assert.assertTrue(files[0].getPath().getName().equals("000000_0")); } Assert.assertEquals(expectedDeltas, sawDeltaTimes); // Verify query result int [][] resultData = new int[][] {{1,2}, {3,4}}; List<String> rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(resultData), rs); } }
runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(3,4)"); status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (TableExtended.MMTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(5,6),(7,8)"); runStatementOnDriver("insert overwrite table " + TableExtended.MMTBL + " select a,b from " + Table.NONACIDORCTBL); status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (TableExtended.MMTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); List<String> rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(resultData), rs);
public void testMmTableCompaction() throws Exception { runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(3,4)"); runStatementOnDriver("alter table "+ TableExtended.MMTBL + " compact 'MINOR'"); runWorker(hiveConf); verifyDirAndResult(2); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(5,6)"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); runStatementOnDriver("alter table "+ TableExtended.MMTBL + " compact 'MINOR'"); runWorker(hiveConf);
runStatementOnDriver("insert into " + TableExtended.MMTBLPART + " PARTITION(p='odd') " + makeValuesClause(valuesOdd)); runStatementOnDriver("insert into " + TableExtended.MMTBLPART + " PARTITION(p='even') " + makeValuesClause(valuesEven)); int[][] newValsEven = {{2,2}}; runStatementOnDriver("insert into " + TableExtended.NONACIDPART + " PARTITION(p='odd') " + makeValuesClause(newValsOdd)); runStatementOnDriver("insert into " + TableExtended.NONACIDPART + " PARTITION(p='even') " + makeValuesClause(newValsEven)); runStatementOnDriver("insert overwrite table " + TableExtended.MMTBLPART + " partition(p) select a,b,p from " + TableExtended.NONACIDPART); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBLPART + " where p='even' order by a,b"); int [][] rExpectedEven = new int[][] {{2,2}}; Assert.assertEquals(stringifyValues(rExpectedEven), rs); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBLPART + " where p='odd' order by a,b"); int [][] rExpectedOdd = new int[][] {{5,5},{11,11}}; Assert.assertEquals(stringifyValues(rExpectedOdd), rs); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBLPART + " order by a,b"); int[][] rExpected = new int[][] {{2,2},{5,5},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), rs);
runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(3,4)"); status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (TableExtended.MMTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + TestTxnCommands2.makeValuesClause(values)); runStatementOnDriver("insert overwrite table " + TableExtended.MMTBL + " select a,b from " + Table.NONACIDORCTBL + " where a between 1 and 3 union all select a,b from " + Table.NONACIDORCTBL + " where a between 5 and 7"); Assert.assertTrue(sawBase); List<String> rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); int[][] rExpected = new int[][] {{1,2},{2,4},{5,6},{6,8}}; Assert.assertEquals(stringifyValues(rExpected), rs); runStatementOnDriver("alter table "+ TableExtended.MMTBL + " compact 'MAJOR'"); runWorker(hiveConf); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(rExpected), rs); Assert.assertTrue(sawBase); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(rExpected), rs);
runStatementOnDriver("insert into " + TableExtended.MMTBLPART + " PARTITION(p='odd') " + makeValuesClause(valuesOdd)); runStatementOnDriver("insert into " + TableExtended.MMTBLPART + " PARTITION(p='even') " + makeValuesClause(valuesEven)); runStatementOnDriver("insert into " + TableExtended.NONACIDPART + " PARTITION(p='odd') " + makeValuesClause(newValsOdd)); runStatementOnDriver("insert into " + TableExtended.NONACIDPART + " PARTITION(p='even') " + makeValuesClause(newValsEven)); String s = "insert overwrite table " + TableExtended.MMTBLPART + " PARTITION(p='odd') " + " select a,b from " + TableExtended.NONACIDPART + " where " + TableExtended.NONACIDPART + ".p='odd'"; rs = runStatementOnDriver("explain formatted " + s); LOG.info("Explain formatted: " + rs.toString()); runStatementOnDriver(s); runStatementOnDriver(s); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBLPART + " where p='even' order by a,b"); int [][] rExpectedEven = new int[][] {{2,2}}; Assert.assertEquals(stringifyValues(rExpectedEven), rs); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBLPART + " where p='odd' order by a,b"); int [][] rExpectedOdd = new int[][] {{5,5},{11,11}}; Assert.assertEquals(stringifyValues(rExpectedOdd), rs); runStatementOnDriver("alter table "+ TableExtended.MMTBLPART + " PARTITION(p='odd') " + " compact 'MAJOR'" ); runWorker(hiveConf); runStatementOnDriver("alter table "+ TableExtended.MMTBLPART + " PARTITION(p='even') " + " compact 'MAJOR'" ); runWorker(hiveConf); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBLPART + " order by a,b"); int[][] rExpected = new int[][] {{2,2},{5,5},{11,11}};
@Test public void testOperationsOnCompletedTxnComponentsForMmTable() throws Exception { // Insert two rows into the table. runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(3,4)"); // There should be 2 delta directories verifyDirAndResult(2); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from COMPLETED_TXN_COMPONENTS"), 2, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS")); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from TXNS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXNS")); // Initiate a minor compaction request on the table. runStatementOnDriver("alter table " + TableExtended.MMTBL + " compact 'MAJOR'"); // Run worker. runWorker(hiveConf); // Run Cleaner. runCleaner(hiveConf); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from COMPLETED_TXN_COMPONENTS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS")); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from TXNS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXNS")); }
runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(3,4)"); runStatementOnDriver("alter table " + TableExtended.MMTBL + " compact 'MINOR'"); runStatementOnDriver("insert into " + TableExtended.MMTBL + " values (5, 6)"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); List<String> rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(expected), rs); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(expected), rs); rs = runStatementOnDriver("select a,b from " + TableExtended.MMTBL + " order by a,b"); Assert.assertEquals(stringifyValues(expected), rs); runStatementOnDriver("alter table " + TableExtended.MMTBL + " compact 'MINOR'");