@VisibleForTesting public Optional<Partition> getPartitionObject(String completePartitionName) { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { List<String> partitionList = At_SPLITTER.splitToList(completePartitionName); if (partitionList.size() != 3) { log.warn("Invalid partition name " + completePartitionName); return Optional.<Partition>absent(); } Partition sourcePartition = client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2)); return Optional.fromNullable(sourcePartition); } catch (IOException | TException e) { log.warn("Unable to get partition object from metastore for partition " + completePartitionName); } return Optional.<Partition>absent(); }
public static HivePartitionDataset findDataset(String completePartitionName, State prop) throws IOException { synchronized (lock) { List<String> partitionList = AT_SPLITTER.splitToList(completePartitionName); Preconditions.checkArgument(partitionList.size() == 3, "Invalid partition name"); if (!pool.isPresent()) { pool = Optional.of(HiveMetastoreClientPool.get(new Properties(), Optional.fromNullable(new Properties().getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY)))); } try (AutoReturnableObject<IMetaStoreClient> client = pool.get().getClient()) { Table table = new Table(client.get().getTable(partitionList.get(0), partitionList.get(1))); Partition partition = new Partition(table, client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2))); return new HivePartitionDataset(partition); } catch (TException | HiveException e) { throw new IOException(e); } } }
@Override public HiveMetastoreClientPool call() throws Exception { return new HiveMetastoreClientPool(properties, metastoreURI); } });
@Override public void dropPartitionIfExists(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(DROP_TABLE).time()) { client.get().dropPartition(dbName, tableName, partitionValues, false); } String metastoreURI = this.clientPool.getHiveConf().get(HiveMetaStoreClientFactory.HIVE_METASTORE_TOKEN_SIGNATURE, "null"); HiveMetaStoreEventHelper.submitSuccessfulPartitionDrop(eventSubmitter, dbName, tableName, partitionValues, metastoreURI); log.info("Dropped partition " + partitionValues + " in table " + tableName + " in db " + dbName); } catch (NoSuchObjectException e) { // Partition does not exist. Nothing to do } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPartitionDrop(eventSubmitter, dbName, tableName, partitionValues, e); throw new IOException(String.format("Unable to check existence of Hive partition %s in table %s in db %s", partitionValues, tableName, dbName), e); } }
protected static HiveMetastoreClientPool createClientPool(Properties properties) throws IOException { return HiveMetastoreClientPool.get(properties, Optional.fromNullable(properties.getProperty(HIVE_METASTORE_URI_KEY))); }
private HiveMetastoreClientPool getTestPool(List<HiveDatasetFinder.DbAndTable> dbAndTables) throws Exception { SetMultimap<String, String> entities = HashMultimap.create(); for (HiveDatasetFinder.DbAndTable dbAndTable : dbAndTables) { entities.put(dbAndTable.getDb(), dbAndTable.getTable()); } HiveMetastoreClientPool pool = Mockito.mock(HiveMetastoreClientPool.class); IMetaStoreClient client = Mockito.mock(IMetaStoreClient.class); Mockito.when(client.getAllDatabases()).thenReturn(Lists.newArrayList(entities.keySet())); for (String db : entities.keySet()) { Mockito.doReturn(Lists.newArrayList(entities.get(db))).when(client).getAllTables(db); } for (HiveDatasetFinder.DbAndTable dbAndTable : dbAndTables) { Table table = new Table(); table.setDbName(dbAndTable.getDb()); table.setTableName(dbAndTable.getTable()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation("/tmp/test"); table.setSd(sd); Mockito.doReturn(table).when(client).getTable(dbAndTable.getDb(), dbAndTable.getTable()); } @SuppressWarnings("unchecked") AutoReturnableObject<IMetaStoreClient> aro = Mockito.mock(AutoReturnableObject.class); Mockito.when(aro.get()).thenReturn(client); Mockito.when(pool.getHiveRegProps()).thenReturn(null); Mockito.when(pool.getClient()).thenReturn(aro); return pool; }
this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY)); this.targetClientPool = HiveMetastoreClientPool.get(this.dataset.getProperties(), this.targetURI); this.targetDatabase = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_DATABASE_KEY)) .or(this.dataset.table.getDbName()); try (HiveMetastoreClientPool.MultiClient multiClient = HiveMetastoreClientPool.safeGetClients(namedPools)) {
@Override public void onRemoval(RemovalNotification<Optional<String>, HiveMetastoreClientPool> notification) { if (notification.getValue() != null) { notification.getValue().close(); } } }).build();
/** * Get a {@link HiveMetastoreClientPool} for the requested metastore URI. Useful for using the same pools across * different classes in the code base. Note that if a pool already exists for that metastore, the max number of * objects available will be unchanged, and it might be lower than requested by this method. * * @param properties {@link Properties} used to generate the pool. * @param metastoreURI URI of the Hive metastore. If absent, use default metastore. * @return a {@link HiveMetastoreClientPool}. * @throws IOException */ public static HiveMetastoreClientPool get(final Properties properties, final Optional<String> metastoreURI) throws IOException { synchronized (HiveMetastoreClientPool.class) { if (poolCache == null) { poolCache = createPoolCache(properties); } } try { return poolCache.get(metastoreURI, new Callable<HiveMetastoreClientPool>() { @Override public HiveMetastoreClientPool call() throws Exception { return new HiveMetastoreClientPool(properties, metastoreURI); } }); } catch (ExecutionException ee) { throw new IOException("Failed to get " + HiveMetastoreClientPool.class.getSimpleName(), ee.getCause()); } }
public ComplianceRetentionJob(Properties properties) { super(properties); try { this.pool = HiveMetastoreClientPool .get(properties, Optional.fromNullable(properties.getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); initDatasetFinder(properties); ProxyUtils.cancelTokens(new State(properties)); } catch (InterruptedException | TException | IOException e) { Throwables.propagate(e); } }
@Override public void dropTableIfExists(String dbName, String tableName) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { boolean tableExists; try (Timer.Context context = this.metricContext.timer(TABLE_EXISTS).time()) { tableExists = client.get().tableExists(dbName, tableName); } if (tableExists) { try (Timer.Context context = this.metricContext.timer(DROP_TABLE).time()) { client.get().dropTable(dbName, tableName); } String metastoreURI = this.clientPool.getHiveConf().get(HiveMetaStoreClientFactory.HIVE_METASTORE_TOKEN_SIGNATURE, "null"); HiveMetaStoreEventHelper.submitSuccessfulTableDrop(eventSubmitter, dbName, tableName, metastoreURI); log.info("Dropped table " + tableName + " in db " + dbName); } } catch (TException e) { HiveMetaStoreEventHelper.submitFailedTableDrop(eventSubmitter, dbName, tableName, e); throw new IOException(String.format("Unable to deregister table %s in db %s", tableName, dbName), e); } }
this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY)); this.targetClientPool = HiveMetastoreClientPool.get(this.dataset.getProperties(), this.targetURI); this.targetDatabase = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_DATABASE_KEY)) .or(this.dataset.table.getDbName()); try (HiveMetastoreClientPool.MultiClient multiClient = HiveMetastoreClientPool.safeGetClients(namedPools)) {
@Override public void onRemoval(RemovalNotification<Optional<String>, HiveMetastoreClientPool> notification) { if (notification.getValue() != null) { notification.getValue().close(); } } }).build();
/** * Get a {@link HiveMetastoreClientPool} for the requested metastore URI. Useful for using the same pools across * different classes in the code base. Note that if a pool already exists for that metastore, the max number of * objects available will be unchanged, and it might be lower than requested by this method. * * @param properties {@link Properties} used to generate the pool. * @param metastoreURI URI of the Hive metastore. If absent, use default metastore. * @return a {@link HiveMetastoreClientPool}. * @throws IOException */ public static HiveMetastoreClientPool get(final Properties properties, final Optional<String> metastoreURI) throws IOException { synchronized (HiveMetastoreClientPool.class) { if (poolCache == null) { poolCache = createPoolCache(properties); } } try { return poolCache.get(metastoreURI, new Callable<HiveMetastoreClientPool>() { @Override public HiveMetastoreClientPool call() throws Exception { return new HiveMetastoreClientPool(properties, metastoreURI); } }); } catch (ExecutionException ee) { throw new IOException("Failed to get " + HiveMetastoreClientPool.class.getSimpleName(), ee.getCause()); } }
@VisibleForTesting public boolean dropPartition(String completePartitionName) { List<String> partitionList = At_SPLITTER.splitToList(completePartitionName); if (partitionList.size() != 3) { log.warn("Invalid partition name " + completePartitionName); return false; } try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { client.get().dropPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2), false); return true; } catch (IOException | TException e) { log.warn("Unable to drop Partition " + completePartitionName); } return false; }
EventSubmitter.submit(Optional.of(this.eventSubmitter), EventConstants.VALIDATION_SETUP_EVENT); this.pool = HiveMetastoreClientPool.get(props, Optional.fromNullable(props.getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); Preconditions.checkArgument(props.containsKey(VALIDATION_TYPE_KEY), "Missing property " + VALIDATION_TYPE_KEY); this.validationType = ValidationType.valueOf(props.getProperty(VALIDATION_TYPE_KEY));
@Override public void dropPartitionIfExists(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(DROP_TABLE).time()) { client.get().dropPartition(dbName, tableName, partitionValues, false); } String metastoreURI = this.clientPool.getHiveConf().get(HiveMetaStoreClientFactory.HIVE_METASTORE_TOKEN_SIGNATURE, "null"); HiveMetaStoreEventHelper.submitSuccessfulPartitionDrop(eventSubmitter, dbName, tableName, partitionValues, metastoreURI); log.info("Dropped partition " + partitionValues + " in table " + tableName + " in db " + dbName); } catch (NoSuchObjectException e) { // Partition does not exist. Nothing to do } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPartitionDrop(eventSubmitter, dbName, tableName, partitionValues, e); throw new IOException(String.format("Unable to check existence of Hive partition %s in table %s in db %s", partitionValues, tableName, dbName), e); } }
@Override public HiveMetastoreClientPool call() throws Exception { return new HiveMetastoreClientPool(properties, metastoreURI); } });
@VisibleForTesting public boolean addPartition(Partition destPartition, String completePartitionName) { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { client.get().add_partition(destPartition); return true; } catch (IOException | TException e) { log.warn("Unable to add Partition " + completePartitionName); } return false; }