How to use

HiveClientCache

in

org.apache.hive.hcatalog.common

/**
 * Get or create a hive client depending on whether it exits in cache or not
 * @param hiveConf The hive configuration
 * @return the client
 * @throws MetaException When HiveMetaStoreClient couldn't be created
 * @throws IOException
 */
public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf)
  throws MetaException, IOException {
 if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){
  // If cache is disabled, don't use it.
  return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf);
 }
 // Singleton behaviour: create the cache instance if required.
 if (hiveClientCache == null) {
  synchronized (IMetaStoreClient.class) {
   if (hiveClientCache == null) {
    hiveClientCache = new HiveClientCache(hiveConf);
   }
  }
 }
 try {
  return hiveClientCache.get(hiveConf);
 } catch (LoginException e) {
  throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e);
 }
}

@Test
public void testCloseAllClients() throws IOException, MetaException, LoginException {
 final HiveClientCache cache = new HiveClientCache(1000);
 HiveClientCache.ICacheableMetaStoreClient client1 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf);
 hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, " "); // URIs are checked for string equivalence, even spaces make them different
 HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf);
 cache.closeAllClientsQuietly();
 assertTrue(client1.isClosed());
 assertTrue(client2.isClosed());
}

/**
 * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is
 * healthy and can be reused
 * @param hiveConf
 * @return the hive client
 * @throws MetaException
 * @throws IOException
 * @throws LoginException
 */
public IMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException {
 final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId());
 ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null;
 // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck
 // is if removalListener closes it. The synchronization takes care that removalListener won't do it
 synchronized (CACHE_TEARDOWN_LOCK) {
  cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
  cacheableHiveMetaStoreClient.acquire();
 }
 if (!cacheableHiveMetaStoreClient.isOpen()) {
  synchronized (CACHE_TEARDOWN_LOCK) {
   hiveCache.invalidate(cacheKey);
   cacheableHiveMetaStoreClient.close();
   cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
   cacheableHiveMetaStoreClient.acquire();
  }
 }
 return cacheableHiveMetaStoreClient;
}

@Test
public void testCacheMiss() throws IOException, MetaException, LoginException {
 HiveClientCache cache = new HiveClientCache(1000);
 IMetaStoreClient client = cache.get(hiveConf);
 assertNotNull(client);
 // Set different uri as it is one of the criteria deciding whether to return the same client or not
 hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, " "); // URIs are checked for string equivalence, even spaces make them different
 IMetaStoreClient client2 = cache.get(hiveConf);
 assertNotNull(client2);
 assertNotSame(client, client2);
}

 @Override
 public void run() {
  LOG.debug("Cleaning up hive client cache in ShutDown hook");
  cleanupHandle.cancel(false); // Cancel the maintenance thread.
  closeAllClientsQuietly();
 }
};

/**
 * Check that a *new* client is created if asked from different threads even with
 * the same hive configuration
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test
public void testMultipleThreadAccess() throws ExecutionException, InterruptedException {
 final HiveClientCache cache = new HiveClientCache(1000);
 class GetHiveClient implements Callable<IMetaStoreClient> {
  @Override
  public IMetaStoreClient call() throws IOException, MetaException, LoginException {
   return cache.get(hiveConf);
  }
 }
 ExecutorService executor = Executors.newFixedThreadPool(2);
 Callable<IMetaStoreClient> worker1 = new GetHiveClient();
 Callable<IMetaStoreClient> worker2 = new GetHiveClient();
 Future<IMetaStoreClient> clientFuture1 = executor.submit(worker1);
 Future<IMetaStoreClient> clientFuture2 = executor.submit(worker2);
 IMetaStoreClient client1 = clientFuture1.get();
 IMetaStoreClient client2 = clientFuture2.get();
 assertNotNull(client1);
 assertNotNull(client2);
 assertNotSame(client1, client2);
}

/**
 * Check that a new client is returned for the same configuration after the expiry time.
 * Also verify that the expiry time configuration is honoured
 */
@Test
public void testCacheExpiry() throws IOException, MetaException, LoginException, InterruptedException {
 HiveClientCache cache = new HiveClientCache(1);
 HiveClientCache.ICacheableMetaStoreClient client = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf);
 assertNotNull(client);
 Thread.sleep(2500);
 HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf);
 client.close();
 assertTrue(client.isClosed()); // close() after *expiry time* and *a cache access* should  have tore down the client
 assertNotNull(client2);
 assertNotSame(client, client2);
}

 @Override
 public void run() {
  LOG.debug("Cleaning up hive client cache in ShutDown hook");
  cleanupHandle.cancel(false); // Cancel the maintenance thread.
  closeAllClientsQuietly();
 }
};

/**
 * Get or create a hive client depending on whether it exits in cache or not
 * @param hiveConf The hive configuration
 * @return the client
 * @throws MetaException When HiveMetaStoreClient couldn't be created
 * @throws IOException
 */
public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf)
  throws MetaException, IOException {
 if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){
  // If cache is disabled, don't use it.
  return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf);
 }
 // Singleton behaviour: create the cache instance if required.
 if (hiveClientCache == null) {
  synchronized (IMetaStoreClient.class) {
   if (hiveClientCache == null) {
    hiveClientCache = new HiveClientCache(hiveConf);
   }
  }
 }
 try {
  return hiveClientCache.get(hiveConf);
 } catch (LoginException e) {
  throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e);
 }
}

@Test
public void testCacheHit() throws IOException, MetaException, LoginException {
 HiveClientCache cache = new HiveClientCache(1000);
 HiveClientCache.ICacheableMetaStoreClient client = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf);
 assertNotNull(client);
 client.close(); // close shouldn't matter
 // Setting a non important configuration should return the same client only
 hiveConf.setIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS, 10);
 HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf);
 assertNotNull(client2);
 assertSame(client, client2);
 assertEquals(client.getUsers(), client2.getUsers());
 client2.close();
}

/**
 * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is
 * healthy and can be reused
 * @param hiveConf
 * @return the hive client
 * @throws MetaException
 * @throws IOException
 * @throws LoginException
 */
public IMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException {
 final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId());
 ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null;
 // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck
 // is if removalListener closes it. The synchronization takes care that removalListener won't do it
 synchronized (CACHE_TEARDOWN_LOCK) {
  cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
  cacheableHiveMetaStoreClient.acquire();
 }
 if (!cacheableHiveMetaStoreClient.isOpen()) {
  synchronized (CACHE_TEARDOWN_LOCK) {
   hiveCache.invalidate(cacheKey);
   cacheableHiveMetaStoreClient.close();
   cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
   cacheableHiveMetaStoreClient.acquire();
  }
 }
 return cacheableHiveMetaStoreClient;
}

 @Override
 public void run() {
  LOG.debug("Cleaning up hive client cache in ShutDown hook");
  cleanupHandle.cancel(false); // Cancel the maintenance thread.
  closeAllClientsQuietly();
 }
};

/**
 * Get or create a hive client depending on whether it exits in cache or not
 * @param hiveConf The hive configuration
 * @return the client
 * @throws MetaException When HiveMetaStoreClient couldn't be created
 * @throws IOException
 */
public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf)
  throws MetaException, IOException {
 if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){
  // If cache is disabled, don't use it.
  return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf);
 }
 // Singleton behaviour: create the cache instance if required.
 if (hiveClientCache == null) {
  synchronized (IMetaStoreClient.class) {
   if (hiveClientCache == null) {
    hiveClientCache = new HiveClientCache(hiveConf);
   }
  }
 }
 try {
  return hiveClientCache.get(hiveConf);
 } catch (LoginException e) {
  throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e);
 }
}

metaServer.start();
final HiveClientCache cache = new HiveClientCache(1000);
HiveClientCache.CacheableHiveMetaStoreClient client =
  (HiveClientCache.CacheableHiveMetaStoreClient) cache.get(metaServer.getHiveConf());

/**
 * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is
 * healthy and can be reused
 * @param hiveConf
 * @return the hive client
 * @throws MetaException
 * @throws IOException
 * @throws LoginException
 */
public ICacheableMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException {
 final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId());
 ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null;
 // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck
 // is if removalListener closes it. The synchronization takes care that removalListener won't do it
 synchronized (CACHE_TEARDOWN_LOCK) {
  cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
  cacheableHiveMetaStoreClient.acquire();
 }
 if (!cacheableHiveMetaStoreClient.isOpen()) {
  synchronized (CACHE_TEARDOWN_LOCK) {
   hiveCache.invalidate(cacheKey);
   cacheableHiveMetaStoreClient.close();
   cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
   cacheableHiveMetaStoreClient.acquire();
  }
 }
 return cacheableHiveMetaStoreClient;
}

 @Override
 public void run() {
  LOG.debug("Cleaning up hive client cache in ShutDown hook");
  cleanupHandle.cancel(false); // Cancel the maintenance thread.
  closeAllClientsQuietly();
 }
};

/**
 * Get or create a hive client depending on whether it exits in cache or not
 * @param hiveConf The hive configuration
 * @return the client
 * @throws MetaException When HiveMetaStoreClient couldn't be created
 * @throws IOException
 */
public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf)
  throws MetaException, IOException {
 if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){
  // If cache is disabled, don't use it.
  return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf);
 }
 // Singleton behaviour: create the cache instance if required.
 if (hiveClientCache == null) {
  synchronized (IMetaStoreClient.class) {
   if (hiveClientCache == null) {
    hiveClientCache = new HiveClientCache(hiveConf);
   }
  }
 }
 try {
  return hiveClientCache.get(hiveConf);
 } catch (LoginException e) {
  throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e);
 }
}

/**
 * Returns a cached client if exists or else creates one, caches and returns it. It also checks that the client is
 * healthy and can be reused
 * @param hiveConf
 * @return the hive client
 * @throws MetaException
 * @throws IOException
 * @throws LoginException
 */
public ICacheableMetaStoreClient get(final HiveConf hiveConf) throws MetaException, IOException, LoginException {
 final HiveClientCacheKey cacheKey = HiveClientCacheKey.fromHiveConf(hiveConf, getThreadId());
 ICacheableMetaStoreClient cacheableHiveMetaStoreClient = null;
 // the hmsc is not shared across threads. So the only way it could get closed while we are doing healthcheck
 // is if removalListener closes it. The synchronization takes care that removalListener won't do it
 synchronized (CACHE_TEARDOWN_LOCK) {
  cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
  cacheableHiveMetaStoreClient.acquire();
 }
 if (!cacheableHiveMetaStoreClient.isOpen()) {
  synchronized (CACHE_TEARDOWN_LOCK) {
   hiveCache.invalidate(cacheKey);
   cacheableHiveMetaStoreClient.close();
   cacheableHiveMetaStoreClient = getOrCreate(cacheKey);
   cacheableHiveMetaStoreClient.acquire();
  }
 }
 return cacheableHiveMetaStoreClient;
}

 @Override
 public void run() {
  LOG.debug("Cleaning up hive client cache in ShutDown hook");
  cleanupHandle.cancel(false); // Cancel the maintenance thread.
  closeAllClientsQuietly();
 }
};

/**
 * Get or create a hive client depending on whether it exits in cache or not
 * @param hiveConf The hive configuration
 * @return the client
 * @throws MetaException When HiveMetaStoreClient couldn't be created
 * @throws IOException
 */
public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf)
  throws MetaException, IOException {
 if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){
  // If cache is disabled, don't use it.
  return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf);
 }
 // Singleton behaviour: create the cache instance if required.
 if (hiveClientCache == null) {
  synchronized (IMetaStoreClient.class) {
   if (hiveClientCache == null) {
    hiveClientCache = new HiveClientCache(hiveConf);
   }
  }
 }
 try {
  return hiveClientCache.get(hiveConf);
 } catch (LoginException e) {
  throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e);
 }
}