/** * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is * healthy and can be reused * @param hiveConf * @return the hive client * @throws MetaException * @throws IOException * @throws LoginException */ public IMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException { final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId()); ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null; // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck // is if removalListener closes it. The synchronization takes care that removalListener won't do it synchronized (CACHE_TEARDOWN_LOCK) { cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } if (!cacheableHiveMetaStoreClient.isOpen()) { synchronized (CACHE_TEARDOWN_LOCK) { hiveCache.invalidate(cacheKey); cacheableHiveMetaStoreClient.close(); cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } } return cacheableHiveMetaStoreClient; }
@Override public void onRemoval(RemovalNotification<HiveClientCacheKey, ICacheableMetaStoreClient> notification) { ICacheableMetaStoreClient hiveMetaStoreClient = notification.getValue(); if (hiveMetaStoreClient != null) { if (LOG.isDebugEnabled()) { LOG.debug("Evicting client: " + Integer.toHexString(System.identityHashCode(hiveMetaStoreClient))); } // TODO: This global lock may not be necessary as all concurrent methods in ICacheableMetaStoreClient // are synchronized. synchronized (CACHE_TEARDOWN_LOCK) { hiveMetaStoreClient.setExpiredFromCache(); hiveMetaStoreClient.tearDownIfUnused(); } } } };
/** * Note: This doesn't check if they are being used or not, meant only to be called during shutdown etc. */ void closeAllClientsQuietly() { try { ConcurrentMap<HiveClientCacheKey, ICacheableMetaStoreClient> elements = hiveCache.asMap(); for (ICacheableMetaStoreClient cacheableHiveMetaStoreClient : elements.values()) { cacheableHiveMetaStoreClient.tearDown(); } } catch (Exception e) { LOG.warn("Clean up of hive clients in the cache failed. Ignored", e); } if (this.enableStats) { LOG.info("Cache statistics after shutdown: size=" + hiveCache.size() + " " + hiveCache.stats()); } }
/** * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is * healthy and can be reused * @param hiveConf * @return the hive client * @throws MetaException * @throws IOException * @throws LoginException */ public ICacheableMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException { final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId()); ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null; // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck // is if removalListener closes it. The synchronization takes care that removalListener won't do it synchronized (CACHE_TEARDOWN_LOCK) { cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } if (!cacheableHiveMetaStoreClient.isOpen()) { synchronized (CACHE_TEARDOWN_LOCK) { hiveCache.invalidate(cacheKey); cacheableHiveMetaStoreClient.close(); cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } } return cacheableHiveMetaStoreClient; }
/** * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is * healthy and can be reused * @param hiveConf * @return the hive client * @throws MetaException * @throws IOException * @throws LoginException */ public ICacheableMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException { final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId()); ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null; // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck // is if removalListener closes it. The synchronization takes care that removalListener won't do it synchronized (CACHE_TEARDOWN_LOCK) { cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } if (!cacheableHiveMetaStoreClient.isOpen()) { synchronized (CACHE_TEARDOWN_LOCK) { hiveCache.invalidate(cacheKey); cacheableHiveMetaStoreClient.close(); cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } } return cacheableHiveMetaStoreClient; }
/** * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is * healthy and can be reused * @param hiveConf * @return the hive client * @throws MetaException * @throws IOException * @throws LoginException */ public ICacheableMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException { final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId()); ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null; // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck // is if removalListener closes it. The synchronization takes care that removalListener won't do it synchronized (CACHE_TEARDOWN_LOCK) { cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } if (!cacheableHiveMetaStoreClient.isOpen()) { synchronized (CACHE_TEARDOWN_LOCK) { hiveCache.invalidate(cacheKey); cacheableHiveMetaStoreClient.close(); cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } } return cacheableHiveMetaStoreClient; }
/** * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is * healthy and can be reused * @param hiveConf * @return the hive client * @throws MetaException * @throws IOException * @throws LoginException */ public IMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException { final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId()); ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null; // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck // is if removalListener closes it. The synchronization takes care that removalListener won't do it synchronized (CACHE_TEARDOWN_LOCK) { cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } if (!cacheableHiveMetaStoreClient.isOpen()) { synchronized (CACHE_TEARDOWN_LOCK) { hiveCache.invalidate(cacheKey); cacheableHiveMetaStoreClient.close(); cacheableHiveMetaStoreClient = getOrCreate(cacheKey); cacheableHiveMetaStoreClient.acquire(); } } return cacheableHiveMetaStoreClient; }
@Override public void onRemoval(RemovalNotification<HiveClientCacheKey, ICacheableMetaStoreClient> notification) { ICacheableMetaStoreClient hiveMetaStoreClient = notification.getValue(); if (hiveMetaStoreClient != null) { if (LOG.isDebugEnabled()) { LOG.debug("Evicting client: " + Integer.toHexString(System.identityHashCode(hiveMetaStoreClient))); } // TODO: This global lock may not be necessary as all concurrent methods in ICacheableMetaStoreClient // are synchronized. synchronized (CACHE_TEARDOWN_LOCK) { hiveMetaStoreClient.setExpiredFromCache(); hiveMetaStoreClient.tearDownIfUnused(); } } } };
/** * Note: This doesn't check if they are being used or not, meant only to be called during shutdown etc. */ void closeAllClientsQuietly() { try { ConcurrentMap<HiveClientCacheKey, ICacheableMetaStoreClient> elements = hiveCache.asMap(); for (ICacheableMetaStoreClient cacheableHiveMetaStoreClient : elements.values()) { cacheableHiveMetaStoreClient.tearDown(); } } catch (Exception e) { LOG.warn("Clean up of hive clients in the cache failed. Ignored", e); } if (this.enableStats) { LOG.info("Cache statistics after shutdown: size=" + hiveCache.size() + " " + hiveCache.stats()); } }
/** * Note: This doesn't check if they are being used or not, meant only to be called during shutdown etc. */ void closeAllClientsQuietly() { try { ConcurrentMap<HiveClientCacheKey, ICacheableMetaStoreClient> elements = hiveCache.asMap(); for (ICacheableMetaStoreClient cacheableHiveMetaStoreClient : elements.values()) { cacheableHiveMetaStoreClient.tearDown(); } } catch (Exception e) { LOG.warn("Clean up of hive clients in the cache failed. Ignored", e); } }
/** * Note: This doesn't check if they are being used or not, meant only to be called during shutdown etc. */ void closeAllClientsQuietly() { try { ConcurrentMap<HiveClientCacheKey, ICacheableMetaStoreClient> elements = hiveCache.asMap(); for (ICacheableMetaStoreClient cacheableHiveMetaStoreClient : elements.values()) { cacheableHiveMetaStoreClient.tearDown(); } } catch (Exception e) { LOG.warn("Clean up of hive clients in the cache failed. Ignored", e); } }
/** * Note: This doesn't check if they are being used or not, meant only to be called during shutdown etc. */ void closeAllClientsQuietly() { try { ConcurrentMap<HiveClientCacheKey, ICacheableMetaStoreClient> elements = hiveCache.asMap(); for (ICacheableMetaStoreClient cacheableHiveMetaStoreClient : elements.values()) { cacheableHiveMetaStoreClient.tearDown(); } } catch (Exception e) { LOG.warn("Clean up of hive clients in the cache failed. Ignored", e); } }
@Override public void onRemoval(RemovalNotification<HiveClientCacheKey, ICacheableMetaStoreClient> notification) { ICacheableMetaStoreClient hiveMetaStoreClient = notification.getValue(); if (hiveMetaStoreClient != null) { synchronized (CACHE_TEARDOWN_LOCK) { hiveMetaStoreClient.setExpiredFromCache(); hiveMetaStoreClient.tearDownIfUnused(); } } } };
@Override public void onRemoval(RemovalNotification<HiveClientCacheKey, ICacheableMetaStoreClient> notification) { ICacheableMetaStoreClient hiveMetaStoreClient = notification.getValue(); if (hiveMetaStoreClient != null) { synchronized (CACHE_TEARDOWN_LOCK) { hiveMetaStoreClient.setExpiredFromCache(); hiveMetaStoreClient.tearDownIfUnused(); } } } };
@Override public void onRemoval(RemovalNotification<HiveClientCacheKey, ICacheableMetaStoreClient> notification) { ICacheableMetaStoreClient hiveMetaStoreClient = notification.getValue(); if (hiveMetaStoreClient != null) { synchronized (CACHE_TEARDOWN_LOCK) { hiveMetaStoreClient.setExpiredFromCache(); hiveMetaStoreClient.tearDownIfUnused(); } } } };
/** * Check that a new client is returned for the same configuration after the expiry time. * Also verify that the expiry time configuration is honoured */ @Test public void testCacheExpiry() throws IOException, MetaException, LoginException, InterruptedException { HiveClientCache cache = new HiveClientCache(1); HiveClientCache.ICacheableMetaStoreClient client = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); assertNotNull(client); Thread.sleep(2500); HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); client.close(); assertTrue(client.isClosed()); // close() after *expiry time* and *a cache access* should have tore down the client assertNotNull(client2); assertNotSame(client, client2); }
@Test public void testCacheHit() throws IOException, MetaException, LoginException { HiveClientCache cache = new HiveClientCache(1000); HiveClientCache.ICacheableMetaStoreClient client = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); assertNotNull(client); client.close(); // close shouldn't matter // Setting a non important configuration should return the same client only hiveConf.setIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS, 10); HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); assertNotNull(client2); assertSame(client, client2); assertEquals(client.getUsers(), client2.getUsers()); client2.close(); }