/** * Close the underlying objects irrespective of whether they are in use or not. */ public void tearDown() { try { if (!isClosed) { super.close(); } isClosed = true; } catch (Exception e) { LOG.warn("Error closing hive metastore client. Ignored.", e); } }
/** * Close the underlying objects irrespective of whether they are in use or not. */ public void tearDown() { try { if (!isClosed) { super.close(); } isClosed = true; } catch (Exception e) { LOG.warn("Error closing hive metastore client. Ignored.", e); } }
@After public void closeClient() { if (hmsc != null) { hmsc.close(); } }
@After public void tearDown() throws Exception { client.close(); }
@After public void tearDown() throws Exception { if (client != null) { client.close(); } }
@Override protected void tearDown() throws Exception { try { super.tearDown(); client.close(); } catch (Throwable e) { System.err.println("Unable to close metastore"); System.err.println(StringUtils.stringifyException(e)); throw new Exception(e); } }
/** * Help method which gets hive tables for a given schema|DB name and a list of table names. * Retries once if the first call fails with TExcption other than connection-lost problems. * @param mClient * @param schemaName * @param tableNames * @return list of hive table instances. **/ public static List<Table> getTableObjectsByNameHelper(final HiveMetaStoreClient mClient, final String schemaName, final List<String> tableNames) throws TException { try { return mClient.getTableObjectsByName(schemaName, tableNames); } catch (MetaException | InvalidOperationException | UnknownDBException e) { throw e; } catch (TException e) { logger.warn("Failure while attempting to get tables by names. Retries once. ", e); try { mClient.close(); } catch (Exception ex) { logger.warn("Failure while attempting to close existing hive metastore connection. May leak connection.", ex); } mClient.reconnect(); return mClient.getTableObjectsByName(schemaName, tableNames); } }
@Override protected void tearDown() throws Exception { try { super.tearDown(); client.dropTable(dbName, tblName); client.dropDatabase(dbName); client.close(); } catch (Throwable e) { LOG.error("Unable to close metastore", e); throw new Exception(e); } }
@AfterClass public static void tearDown() throws Exception { HiveMetaStore.TEST_TIMEOUT_ENABLED = false; try { client.close(); } catch (Throwable e) { System.err.println("Unable to close metastore"); System.err.println(StringUtils.stringifyException(e)); throw e; } }
@Override public void reconnect() throws MetaException { if (localMetaStore) { // For direct DB connections we don't yet support reestablishing connections. throw new MetaException("For direct MetaStore DB connections, we don't support retries" + " at the client level."); } else { close(); if (uriResolverHook != null) { //for dynamic uris, re-lookup if there are new metastore locations resolveUris(); } if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { // Swap the first element of the metastoreUris[] with a random element from the rest // of the array. Rationale being that this method will generally be called when the default // connection has died and the default connection is likely to be the first array element. promoteRandomMetaStoreURI(); } open(); } }
@After public void cleanup() { dropTestTables(); client.close(); client = null; }
void run() throws Exception { wh = new Warehouse(conf); checkOldWarehouseRoot(); checkExternalWarehouseDir(); checkOwnerPermsOptions(); hms = new HiveMetaStoreClient(conf);//MetaException try { List<String> databases = hms.getAllDatabases();//TException LOG.info("Found {} databases", databases.size()); for (String dbName : databases) { if (dbName.matches(runOptions.dbRegex)) { try { processDatabase(dbName); } catch (Exception err) { LOG.error("Error processing database " + dbName, err); failuresEncountered = true; } } } LOG.info("Done processing databases."); } finally { hms.close(); } if (failuresEncountered) { throw new HiveException("One or more failures encountered during processing."); } if (failedValidationChecks) { throw new HiveException("One or more tables failed validation checks for strict managed table mode."); } }
@Test public void testMetaConfDuplicateNotification() throws Exception { HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf, null); closingClient.setMetaConf(metaConfKey, metaConfVal); int beforeCloseNotificationEventCounts = DummyListener.notifyList.size(); closingClient.close(); Thread.sleep(2 * 1000); int afterCloseNotificationEventCounts = DummyListener.notifyList.size(); // Setting key to same value, should not trigger configChange event during shutdown assertEquals(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts); }
@Test public void testMetaConfDuplicateNotification() throws Exception { HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf, null); closingClient.setMetaConf(metaConfKey, metaConfVal); int beforeCloseNotificationEventCounts = DummyListener.notifyList.size(); closingClient.close(); Thread.sleep(2 * 1000); int afterCloseNotificationEventCounts = DummyListener.notifyList.size(); // Setting key to same value, should not trigger configChange event during shutdown assertEquals(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts); }
private static void initializeSetup(HiveConf metastoreConf) throws Exception { hiveConf = new HiveConf(metastoreConf, TestHCatMultiOutputFormat.class); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); hiveConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, HCatSemanticAnalyzer.class.getName()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.WAREHOUSE)); System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.THRIFT_URIS)); hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousedir.toString()); try { hmsc = new HiveMetaStoreClient(hiveConf); initalizeTables(); } catch (Throwable e) { LOG.error("Exception encountered while setting up testcase", e); throw new Exception(e); } finally { hmsc.close(); } }
@Test public void testMetaConfSameHandler() throws Exception { HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf, null); closingClient.setMetaConf(metaConfKey, "[test pattern modified]"); ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent(); int beforeCloseNotificationEventCounts = DummyListener.notifyList.size(); IHMSHandler beforeHandler = event.getIHMSHandler(); closingClient.close(); Thread.sleep(2 * 1000); event = (ConfigChangeEvent) DummyListener.getLastEvent(); int afterCloseNotificationEventCounts = DummyListener.notifyList.size(); IHMSHandler afterHandler = event.getIHMSHandler(); // Meta-conf cleanup should trigger an event to listener assertNotSame(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts); // Both the handlers should be same assertEquals(beforeHandler, afterHandler); } }
@Test public void testMetaConfSameHandler() throws Exception { HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf, null); closingClient.setMetaConf(metaConfKey, "[test pattern modified]"); ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent(); int beforeCloseNotificationEventCounts = DummyListener.notifyList.size(); IHMSHandler beforeHandler = event.getHandler(); closingClient.close(); Thread.sleep(2 * 1000); event = (ConfigChangeEvent) DummyListener.getLastEvent(); int afterCloseNotificationEventCounts = DummyListener.notifyList.size(); IHMSHandler afterHandler = event.getHandler(); // Meta-conf cleanup should trigger an event to listener assertNotSame(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts); // Both the handlers should be same assertEquals(beforeHandler, afterHandler); } }
@Test public void testJDOPersistanceManagerCleanup() throws Exception { if (isThriftClient == false) { return; } int numObjectsBeforeClose = getJDOPersistanceManagerCacheSize(); HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf); closingClient.getAllDatabases(); closingClient.close(); Thread.sleep(5 * 1000); // give HMS time to handle close request int numObjectsAfterClose = getJDOPersistanceManagerCacheSize(); assertTrue(numObjectsBeforeClose == numObjectsAfterClose); HiveMetaStoreClient nonClosingClient = new HiveMetaStoreClient(conf); nonClosingClient.getAllDatabases(); // Drop connection without calling close. HMS thread deleteContext // will trigger cleanup nonClosingClient.getTTransport().close(); Thread.sleep(5 * 1000); int numObjectsAfterDroppedConnection = getJDOPersistanceManagerCacheSize(); assertTrue(numObjectsAfterClose == numObjectsAfterDroppedConnection); }
@Test public void testMetaConfNotifyListenersClosingClient() throws Exception { HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf, null); closingClient.setMetaConf(metaConfKey, "[test pattern modified]"); ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent(); assertEquals(event.getOldValue(), metaConfVal); assertEquals(event.getNewValue(), "[test pattern modified]"); closingClient.close(); Thread.sleep(2 * 1000); event = (ConfigChangeEvent) DummyListener.getLastEvent(); assertEquals(event.getOldValue(), "[test pattern modified]"); assertEquals(event.getNewValue(), metaConfVal); }
@Test public void testMetaConfNotifyListenersClosingClient() throws Exception { HiveMetaStoreClient closingClient = new HiveMetaStoreClient(conf, null); closingClient.setMetaConf(metaConfKey, "[test pattern modified]"); ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent(); assertEquals(event.getOldValue(), metaConfVal); assertEquals(event.getNewValue(), "[test pattern modified]"); closingClient.close(); Thread.sleep(2 * 1000); event = (ConfigChangeEvent) DummyListener.getLastEvent(); assertEquals(event.getOldValue(), "[test pattern modified]"); assertEquals(event.getNewValue(), metaConfVal); }