/** * Get the {@link IMetaStoreClient} with the provided name. * @throws IOException */ public IMetaStoreClient getClient(String name) throws IOException { if (!this.clients.containsKey(name)) { throw new IOException("There is no client with name " + name); } return this.clients.get(name).get(); }
@VisibleForTesting public Optional<Partition> getPartitionObject(String completePartitionName) { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { List<String> partitionList = At_SPLITTER.splitToList(completePartitionName); if (partitionList.size() != 3) { log.warn("Invalid partition name " + completePartitionName); return Optional.<Partition>absent(); } Partition sourcePartition = client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2)); return Optional.fromNullable(sourcePartition); } catch (IOException | TException e) { log.warn("Unable to get partition object from metastore for partition " + completePartitionName); } return Optional.<Partition>absent(); }
@VisibleForTesting public boolean addPartition(Partition destPartition, String completePartitionName) { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { client.get().add_partition(destPartition); return true; } catch (IOException | TException e) { log.warn("Unable to add Partition " + completePartitionName); } return false; }
@VisibleForTesting public boolean dropPartition(String completePartitionName) { List<String> partitionList = At_SPLITTER.splitToList(completePartitionName); if (partitionList.size() != 3) { log.warn("Invalid partition name " + completePartitionName); return false; } try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { client.get().dropPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2), false); return true; } catch (IOException | TException e) { log.warn("Unable to drop Partition " + completePartitionName); } return false; }
@Override public boolean createDbIfNotExists(String dbName) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { return createDbIfNotExists(client.get(), dbName); } }
@Override public boolean existsTable(String dbName, String tableName) throws IOException { if (this.optimizedChecks && this.tableAndDbExistenceCache.getIfPresent(dbName + ":" + tableName ) != null ) { return true; } try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(TABLE_EXISTS).time()) { return client.get().tableExists(dbName, tableName); } } catch (TException e) { throw new IOException(String.format("Unable to check existence of table %s in db %s", tableName, dbName), e); } }
private Pair<Optional<org.apache.hadoop.hive.metastore.api.Table>, Optional<List<Partition>>> getDestinationTableMeta(String dbName, String tableName, Properties props) { Optional<org.apache.hadoop.hive.metastore.api.Table> table = Optional.absent(); Optional<List<Partition>> partitions = Optional.absent(); try { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { table = Optional.of(client.get().getTable(dbName, tableName)); if (table.isPresent()) { org.apache.hadoop.hive.ql.metadata.Table qlTable = new org.apache.hadoop.hive.ql.metadata.Table(table.get()); if (HiveUtils.isPartitioned(qlTable)) { partitions = Optional.of(HiveUtils.getPartitions(client.get(), qlTable, Optional.<String> absent())); } } } } catch (NoSuchObjectException e) { return ImmutablePair.of(table, partitions); } catch (IOException | TException e) { throw new RuntimeException("Could not fetch destination table metadata", e); } return ImmutablePair.of(table, partitions); } }
@Override public boolean existsPartition(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(GET_HIVE_PARTITION).time()) { client.get().getPartition(dbName, tableName, partitionValues); } return true; } catch (NoSuchObjectException e) { return false; } catch (TException e) { throw new IOException(String.format("Unable to check existence of partition %s in table %s in db %s", partitionValues, tableName, dbName), e); } }
@Override public Optional<HiveTable> getTable(String dbName, String tableName) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Table hiveTable; try (Timer.Context context = this.metricContext.timer(GET_HIVE_TABLE).time()) { hiveTable = client.get().getTable(dbName, tableName); } return Optional.of(HiveMetaStoreUtils.getHiveTable(hiveTable)); } catch (NoSuchObjectException e) { return Optional.<HiveTable> absent(); } catch (TException e) { throw new IOException("Unable to get table " + tableName + " in db " + dbName, e); } }
@Override public Optional<HivePartition> getPartition(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Partition hivePartition; try (Timer.Context context = this.metricContext.timer(GET_HIVE_PARTITION).time()) { hivePartition = client.get().getPartition(dbName, tableName, partitionValues); } return Optional.of(HiveMetaStoreUtils.getHivePartition(hivePartition)); } catch (NoSuchObjectException e) { return Optional.<HivePartition> absent(); } catch (TException e) { throw new IOException( "Unable to get partition " + partitionValues + " from table " + tableName + " in db " + dbName, e); } }
public HiveMetadataForCompactionExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException { super(state); if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) { log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY)); return; } try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) { Table table = client.get().getTable(this.dbName, this.tableName); String primaryKeyString = table.getParameters().get(state.getProp(COMPACTION_PRIMARY_KEY)); List<String> primaryKeyList = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(primaryKeyString); String deltaString = table.getParameters().get(state.getProp(COMPACTION_DELTA)); List<String> deltaList = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(deltaString); Path dataFilesPath = new Path(table.getSd().getLocation()); compactionEntity = new MRCompactionEntity(primaryKeyList, deltaList, dataFilesPath, state.getProperties()); } }
/** * This method returns a sorted list of partitions. */ public List<Partition> getPartitionsFromDataset() throws IOException{ try (AutoReturnableObject<IMetaStoreClient> client = getClientPool().getClient()) { List<Partition> partitions = HiveUtils.getPartitions(client.get(), getTable(), Optional.<String>absent()); return sortPartitions(partitions); } }
private Pair<Optional<Table>, Optional<List<Partition>>> getDestinationTableMeta(String dbName, String tableName, WorkUnitState state) throws DataConversionException { Optional<Table> table = Optional.<Table>absent(); Optional<List<Partition>> partitions = Optional.<List<Partition>>absent(); try { HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { table = Optional.of(client.get().getTable(dbName, tableName)); if (table.isPresent()) { org.apache.hadoop.hive.ql.metadata.Table qlTable = new org.apache.hadoop.hive.ql.metadata.Table(table.get()); if (HiveUtils.isPartitioned(qlTable)) { partitions = Optional.of(HiveUtils.getPartitions(client.get(), qlTable, Optional.<String>absent())); } } } } catch (NoSuchObjectException e) { return ImmutablePair.of(table, partitions); } catch (IOException | TException e) { throw new DataConversionException("Could not fetch destination table metadata", e); } return ImmutablePair.of(table, partitions); }
@Override protected void registerPath(HiveSpec spec) throws IOException { try (Timer.Context context = this.metricContext.timer(PATH_REGISTER_TIMER).time(); AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Table table = HiveMetaStoreUtils.getTable(spec.getTable()); createDbIfNotExists(client.get(), table.getDbName()); createOrAlterTable(client.get(), table, spec); Optional<HivePartition> partition = spec.getPartition(); if (partition.isPresent()) { addOrAlterPartition(client.get(), table, HiveMetaStoreUtils.getPartition(partition.get()), spec); } HiveMetaStoreEventHelper.submitSuccessfulPathRegistration(eventSubmitter, spec); } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPathRegistration(eventSubmitter, spec, e); throw new IOException(e); } }
@Override public void dropTableIfExists(String dbName, String tableName) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { boolean tableExists; try (Timer.Context context = this.metricContext.timer(TABLE_EXISTS).time()) { tableExists = client.get().tableExists(dbName, tableName); } if (tableExists) { try (Timer.Context context = this.metricContext.timer(DROP_TABLE).time()) { client.get().dropTable(dbName, tableName); } String metastoreURI = this.clientPool.getHiveConf().get(HiveMetaStoreClientFactory.HIVE_METASTORE_TOKEN_SIGNATURE, "null"); HiveMetaStoreEventHelper.submitSuccessfulTableDrop(eventSubmitter, dbName, tableName, metastoreURI); log.info("Dropped table " + tableName + " in db " + dbName); } } catch (TException e) { HiveMetaStoreEventHelper.submitFailedTableDrop(eventSubmitter, dbName, tableName, e); throw new IOException(String.format("Unable to deregister table %s in db %s", tableName, dbName), e); } }
@Override public void dropPartitionIfExists(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(DROP_TABLE).time()) { client.get().dropPartition(dbName, tableName, partitionValues, false); } String metastoreURI = this.clientPool.getHiveConf().get(HiveMetaStoreClientFactory.HIVE_METASTORE_TOKEN_SIGNATURE, "null"); HiveMetaStoreEventHelper.submitSuccessfulPartitionDrop(eventSubmitter, dbName, tableName, partitionValues, metastoreURI); log.info("Dropped partition " + partitionValues + " in table " + tableName + " in db " + dbName); } catch (NoSuchObjectException e) { // Partition does not exist. Nothing to do } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPartitionDrop(eventSubmitter, dbName, tableName, partitionValues, e); throw new IOException(String.format("Unable to check existence of Hive partition %s in table %s in db %s", partitionValues, tableName, dbName), e); } }
@Override public boolean createTableIfNotExists(HiveTable table) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient(); AutoCloseableLock lock = this.locks.getTableLock(table.getDbName(), table.getTableName())) { return createTableIfNotExists(client.get(), HiveMetaStoreUtils.getTable(table), table); } }
@Override public void clean() throws IOException { // Possible empty directories to clean for this partition (version) Set<Path> possiblyEmptyDirectories = new HashSet<>(); try (AutoReturnableObject<IMetaStoreClient> client = cleanableHiveDataset.getClientPool().getClient()) { Partition partition = hiveDatasetVersion.getPartition(); try { if (!cleanableHiveDataset.isSimulate()) { client.get().dropPartition(partition.getTable().getDbName(), partition.getTable().getTableName(), partition.getValues(), false); log.info("Successfully dropped partition " + partition.getCompleteName()); } else { log.info("Simulating drop partition " + partition.getCompleteName()); } if (cleanableHiveDataset.isShouldDeleteData()) { cleanableHiveDataset.getFsCleanableHelper().clean(hiveDatasetVersion, possiblyEmptyDirectories); } } catch (TException | IOException e) { log.warn(String.format("Failed to completely delete partition %s.", partition.getCompleteName()), e); throw new IOException(e); } } cleanableHiveDataset.getFsCleanableHelper().cleanEmptyDirectories(possiblyEmptyDirectories, cleanableHiveDataset); }
@Override public void alterPartition(HiveTable table, HivePartition partition) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(ALTER_PARTITION).time()) { client.get().alter_partition(table.getDbName(), table.getTableName(), getPartitionWithCreateTimeNow(HiveMetaStoreUtils.getPartition(partition))); } HiveMetaStoreEventHelper.submitSuccessfulPartitionAlter(eventSubmitter, table, partition); } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPartitionAlter(eventSubmitter, table, partition, e); throw new IOException(String.format("Unable to alter partition %s in table %s in db %s", partition.getValues(), table.getTableName(), table.getDbName()), e); } }
@Override public boolean addPartitionIfNotExists(HiveTable table, HivePartition partition) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient(); AutoCloseableLock lock = this.locks.getTableLock(table.getDbName(), table.getTableName())) { try { try (Timer.Context context = this.metricContext.timer(GET_HIVE_PARTITION).time()) { client.get().getPartition(table.getDbName(), table.getTableName(), partition.getValues()); } return false; } catch (NoSuchObjectException e) { try (Timer.Context context = this.metricContext.timer(ALTER_PARTITION).time()) { client.get().alter_partition(table.getDbName(), table.getTableName(), getPartitionWithCreateTimeNow(HiveMetaStoreUtils.getPartition(partition))); } HiveMetaStoreEventHelper.submitSuccessfulPartitionAdd(this.eventSubmitter, table, partition); return true; } } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPartitionAdd(this.eventSubmitter, table, partition, e); throw new IOException(String.format("Unable to add partition %s in table %s in db %s", partition.getValues(), table.getTableName(), table.getDbName()), e); } }