/** * Dump all data in the table by bucket in JSON format */ private void dumpTableData(Table table, long writeId, int stmtId) throws Exception { for(int bucketNum = 0; bucketNum < BUCKET_COUNT; bucketNum++) { dumpBucketData(table, writeId, stmtId, bucketNum); } } @Test
@Test public void testMultipleInserts() throws Exception { runStatementOnDriver("START TRANSACTION"); int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows2)); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match before commit rs", allData, rs); runStatementOnDriver("commit"); dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 1, 1); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match after commit rs1", allData, rs1); } @Test
@Test public void testParallelTruncateAnalyzeStats() throws Exception { String tableName = "mm_table"; List<ColumnStatisticsObj> stats; IMetaStoreClient msClient = prepareParallelTest(tableName, 0); String[] queries = { String.format("truncate table %s", tableName), String.format("analyze table %s compute statistics for columns", tableName) }; runParallelQueries(queries); // Verify stats are either invalid, or valid and correct. stats = getTxnTableStats(msClient, tableName); boolean hasStats = 0 != stats.size(); if (hasStats) { verifyLongStats(0, 0, 0, stats); } // Stats should be valid after analyze. runStatementOnDriver(String.format("analyze table %s compute statistics for columns", tableName)); verifyLongStats(0, 0, 0, getTxnTableStats(msClient, tableName)); }
@Test public void testDeleteIn() throws Exception { runStatementOnDriver("delete from " + Table.ACIDTBL + " where a IN (SELECT A.a from " + Table.ACIDTBL + " A)"); int[][] tableData = {{1,2},{3,2},{5,2},{1,3},{3,3},{5,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("insert into " + Table.ACIDTBL2 + "(a,b,c) values(1,7,17),(3,7,17)"); // runStatementOnDriver("select b from " + Table.ACIDTBL + " where a in (select b from " + Table.NONACIDORCTBL + ")"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a in(select a from " + Table.ACIDTBL2 + ")"); // runStatementOnDriver("delete from " + Table.ACIDTBL + " where a in(select a from " + Table.NONACIDORCTBL + ")"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) select a,b from " + Table.ACIDTBL2); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData = {{1,7},{3,7},{5,2},{5,3}}; Assert.assertEquals("Bulk update failed", stringifyValues(updatedData), rs); } @Test
/** * see https://issues.apache.org/jira/browse/HIVE-14949 for details * @throws Exception */ @Test public void testMergeCardinalityViolation() throws Exception { int[][] sourceVals = {{2,2},{2,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(sourceVals)); int[][] targetVals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(targetVals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED and s.a < 5 THEN DELETE " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) "; runStatementOnDriverNegative(query); runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p) values(1,1,'p1'),(2,2,'p1'),(3,3,'p1'),(4,4,'p2')"); query = "merge into " + Table.ACIDTBLPART + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED and s.a < 5 THEN DELETE " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b, 'p1') "; runStatementOnDriverNegative(query); } @Test
@Test public void testMmExim() throws Exception { String tableName = "mm_table", importName = tableName + "_import"; runStatementOnDriver("drop table if exists " + tableName); runStatementOnDriver(String.format("create table %s (a int, b int) stored as orc " + "TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')", tableName)); runStatementOnDriver(String.format("insert into %s (a,b) %s", tableName, makeValuesClause(rows1))); runStatementOnDriver(String.format("insert into %s (a,b) %s", tableName, makeValuesClause(rows1))); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); org.apache.hadoop.hive.metastore.api.Table table = msClient.getTable("default", tableName); Path exportPath = new Path(table.getSd().getLocation() + "_export"); fs.delete(exportPath, true); runStatementOnDriver(String.format("export table %s to '%s'", tableName, exportPath)); List<String> paths = listPathsRecursive(fs, exportPath); verifyMmExportPaths(paths, 2); runStatementOnDriver(String.format("import table %s from '%s'", importName, exportPath)); org.apache.hadoop.hive.metastore.api.Table imported = msClient.getTable("default", importName); Assert.assertEquals(imported.toString(), "insert_only", FileStatus[] stat = fs.listStatus(importPath, AcidUtils.hiddenFileFilter); Assert.assertEquals(Arrays.toString(stat), 1, stat.length); assertIsDelta(stat[0]); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows1)); allData.sort(null);
@Test public void testTimeOutReaper() throws Exception { runStatementOnDriver("start transaction"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 5"); CommandProcessorResponse cpr = runStatementOnDriverNegative("delete from " + Table.ACIDTBL + " where a = 5"); Assert.assertTrue("Actual: " + cpr.getErrorMessage(), cpr.getErrorMessage().contains("Transaction manager has aborted the transaction txnid:1")); runStatementOnDriver("start transaction"); runStatementOnDriver("select count(*) from " + Table.ACIDTBL + " where a = 17"); pause(750); pause(750); houseKeeperService.run(); pause(750); slr = txnHandler.showLocks(new ShowLocksRequest()); Assert.assertEquals("Unexpected lock count: " + slr, 1, slr.getLocks().size()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); pause(750); houseKeeperService.run(); slr = txnHandler.showLocks(new ShowLocksRequest()); lastHeartbeat < Long.parseLong(vals[1])); runStatementOnDriver("rollback"); slr = txnHandler.showLocks(new ShowLocksRequest()); Assert.assertEquals("Unexpected lock count", 0, slr.getLocks().size());
private IMetaStoreClient prepareParallelTest(String tableName, int val) throws Exception, MetaException, TException, NoSuchObjectException { hiveConf.setBoolean("hive.stats.autogather", true); hiveConf.setBoolean("hive.stats.column.autogather", true); runStatementOnDriver("drop table if exists " + tableName); runStatementOnDriver(String.format("create table %s (a int) stored as orc " + "TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')", tableName)); runStatementOnDriver(String.format("insert into %s (a) values (" + val + ")", tableName)); runStatementOnDriver(String.format("insert into %s (a) values (" + val + ")", tableName)); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); // Stats should be valid after serial inserts. List<ColumnStatisticsObj> stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(1, stats.size()); return msClient; }
public void testNonAcidToAcidConversion01() throws Exception { runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(0,12),(1,5)"); runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true')"); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,17)"); {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t1\t17", "nonacidorctbl/delta_10000001_10000001_0000/bucket_00001"} }; checkResult(expected, query, isVectorized, "before compact", LOG); runStatementOnDriver("alter table "+ TestTxnCommands2.Table.NONACIDORCTBL +" compact 'major'"); TestTxnCommands2.runWorker(hiveConf); {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t1\t17", "nonacidorctbl/base_10000001_v0000020/bucket_00001"} }; checkResult(expected2, query, isVectorized, "after major compact", LOG);
@Test public void testVersioning() throws Exception { hiveConf.set(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getVarname(), "true"); runStatementOnDriver("drop table if exists T"); runStatementOnDriver("create table T (a int, b int) stored as orc"); int[][] data = {{1, 2}}; runStatementOnDriver("insert into T" + makeValuesClause(data)); List<String> rs = runStatementOnDriver("select distinct INPUT__FILE__NAME from T"); FileSystem fs = FileSystem.get(hiveConf); Assert.assertTrue(rs != null && rs.size() == 1 && rs.get(0).contains(AcidUtils.DELTA_PREFIX)); AcidUtils.OrcAcidVersion.ORC_ACID_VERSION, versionFromMetaFile); runStatementOnDriver("insert into T" + makeValuesClause(data)); runStatementOnDriver("alter table T compact 'major'"); TestTxnCommands2.runWorker(hiveConf); Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local")); rs = runStatementOnDriver("select distinct INPUT__FILE__NAME from T"); Assert.assertTrue(rs != null && rs.size() == 1 && rs.get(0).contains(AcidUtils.BASE_PREFIX));
@Override void initHiveConf() { super.initHiveConf(); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); } }
@Test public void testMergeDeleteUpdate() throws Exception { int[][] sourceVals = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(sourceVals)); int[][] targetVals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(targetVals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED and s.a < 5 THEN DELETE " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) "; runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{5,6},{7,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); }
hiveConf.setBoolean("hive.stats.autogather", true); hiveConf.setBoolean("hive.stats.column.autogather", true); runStatementOnDriver("drop table if exists " + tableName); runStatementOnDriver(String.format("create table %s (a int) stored as orc " + "TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')", tableName)); runStatementOnDriver(String.format("insert into %s (a) values (1)", tableName)); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); List<ColumnStatisticsObj> stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(1, stats.size()); runStatementOnDriver(String.format("insert into %s (a) values (1)", tableName)); stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(1, stats.size()); msClient.close(); stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(0, stats.size()); msClient.close(); hiveConf.setBoolean(MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED.getVarname(), true); msClient = new HiveMetaStoreClient(hiveConf); stats = getTxnTableStats(msClient, tableName); runStatementOnDriver(String.format("insert into %s (a) values (1)", tableName)); hiveConf.setBoolean(MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED.getVarname(), true); msClient = new HiveMetaStoreClient(hiveConf); stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(0, stats.size()); msClient.close();
@Test public void testUpdateDeleteOfInserts() throws Exception { int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows2)); Assert.assertEquals("Content didn't match rs1", allData, rs1); runStatementOnDriver("update " + Table.ACIDTBL + " set b = 1 where b != 1"); int[][] updatedData = {{1,1},{3,1},{5,1},{7,1}}; List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after update", stringifyValues(updatedData), rs2); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 7 and b = 1"); dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 2, 0); dumpTableData(Table.ACIDTBL, 2, 2); dumpTableData(Table.ACIDTBL, 2, 4); int[][] updatedData2 = {{1,1},{3,1},{5,1}}; List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3); runStatementOnDriver("commit"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4); } @Test
@Test public void testParallelInsertAnalyzeStats() throws Exception { String tableName = "mm_table"; List<ColumnStatisticsObj> stats; IMetaStoreClient msClient = prepareParallelTest(tableName, 0); String[] queries = { String.format("insert into %s (a) values (999)", tableName), String.format("analyze table %s compute statistics for columns", tableName) }; runParallelQueries(queries); // Verify stats are either invalid, or valid and correct. stats = getTxnTableStats(msClient, tableName); boolean hasStats = 0 != stats.size(); if (hasStats) { verifyLongStats(2, 0, 999, stats); } runStatementOnDriver(String.format("insert into %s (a) values (1000)", tableName)); if (!hasStats) { // Stats should still be invalid if they were invalid. stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(0, stats.size()); } // Stats should be valid after analyze. runStatementOnDriver(String.format("analyze table %s compute statistics for columns", tableName)); verifyLongStats(3, 0, 1000, getTxnTableStats(msClient, tableName)); }
@Test public void testMergeUpdateDelete() throws Exception { int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + //updates (2,1) -> (2,0) "WHEN MATCHED and t.a > 3 and t.a < 5 THEN DELETE " +//deletes (4,3) "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) ";//inserts (11,11) runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,0},{5,6},{7,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); } @Test
@Test public void testMultipleDelete() throws Exception { int[][] rows1 = {{1,2},{3,4},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 8"); int[][] updatedData2 = {{1,2},{3,4},{5,6}}; List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs2); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); int[][] updatedData3 = {{1, 2}, {5, 6}}; List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete2", stringifyValues(updatedData3), rs3); runStatementOnDriver("update " + Table.ACIDTBL + " set b=3"); dumpTableData(Table.ACIDTBL, 1, 0); //nothing actually hashes to bucket0, so update/delete deltas don't have it dumpTableData(Table.ACIDTBL, 2, 0); dumpTableData(Table.ACIDTBL, 2, 2); dumpTableData(Table.ACIDTBL, 2, 4); List<String> rs5 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int [][] updatedData4 = {{1,3},{5,3}}; Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData4), rs5); runStatementOnDriver("commit"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData4), rs4); } @Test
@Test public void testParallelInsertStats() throws Exception { final int TASK_COUNT = 4; String tableName = "mm_table"; List<ColumnStatisticsObj> stats; IMetaStoreClient msClient = prepareParallelTest(tableName, 0); String[] queries = new String[TASK_COUNT]; for (int i = 0; i < queries.length; ++i) { queries[i] = String.format("insert into %s (a) values (" + i + ")", tableName); } runParallelQueries(queries); // Verify stats are either invalid, or valid and correct. stats = getTxnTableStats(msClient, tableName); boolean hasStats = 0 != stats.size(); if (hasStats) { verifyLongStats(TASK_COUNT, 0, TASK_COUNT - 1, stats); } runStatementOnDriver(String.format("insert into %s (a) values (" + TASK_COUNT + ")", tableName)); if (!hasStats) { // Stats should still be invalid if they were invalid. stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(0, stats.size()); } // Stats should be valid after analyze. runStatementOnDriver(String.format("analyze table %s compute statistics for columns", tableName)); verifyLongStats(TASK_COUNT + 1, 0, TASK_COUNT, getTxnTableStats(msClient, tableName)); }
@Test public void testDelete() throws Exception { int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); int[][] updatedData2 = {{1,2}}; List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3); runStatementOnDriver("commit"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4); }
@Test public void testSimpleAcidInsert() throws Exception { int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); //List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); //Assert.assertEquals("Data didn't match in autocommit=true (rs)", stringifyValues(rows1), rs); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows2)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Data didn't match inside tx (rs0)", allData, rs0); runStatementOnDriver("COMMIT WORK"); dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 2, 0); runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); CommandProcessorResponse cpr = runStatementOnDriverNegative("COMMIT");//txn started implicitly by previous statement Assert.assertEquals("Error didn't match: " + cpr, ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr.getErrorCode()); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Data didn't match inside tx (rs0)", allData, rs1); }