@Override public Map<String, Optional<Partition>> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { return loadValue( partitionsByNamesCache, getHivePartitionNames(databaseName, tableName, ImmutableSet.copyOf(partitionNames)), () -> delegate.getPartitionsByNames(databaseName, tableName, partitionNames)); }
private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(Iterable<? extends HivePartitionName> partitionNames) { requireNonNull(partitionNames, "partitionNames is null"); checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty"); HivePartitionName firstPartition = Iterables.get(partitionNames, 0); HiveTableName hiveTableName = firstPartition.getHiveTableName(); String databaseName = hiveTableName.getDatabaseName(); String tableName = hiveTableName.getTableName(); List<String> partitionsToFetch = new ArrayList<>(); for (HivePartitionName partitionName : partitionNames) { checkArgument(partitionName.getHiveTableName().equals(hiveTableName), "Expected table name %s but got %s", hiveTableName, partitionName.getHiveTableName()); partitionsToFetch.add(partitionName.getPartitionName().get()); } ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions = ImmutableMap.builder(); Map<String, Optional<Partition>> partitionsByNames = delegate.getPartitionsByNames(databaseName, tableName, partitionsToFetch); for (Entry<String, Optional<Partition>> entry : partitionsByNames.entrySet()) { partitions.put(HivePartitionName.hivePartitionName(hiveTableName, entry.getKey()), entry.getValue()); } return partitions.build(); }
public synchronized Map<String, Optional<Partition>> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { checkReadable(); TableSource tableSource = getTableSource(databaseName, tableName); Map<List<String>, Action<PartitionAndMore>> partitionActionsOfTable = partitionActions.computeIfAbsent(new SchemaTableName(databaseName, tableName), k -> new HashMap<>()); ImmutableList.Builder<String> partitionNamesToQuery = ImmutableList.builder(); ImmutableMap.Builder<String, Optional<Partition>> resultBuilder = ImmutableMap.builder(); for (String partitionName : partitionNames) { List<String> partitionValues = toPartitionValues(partitionName); Action<PartitionAndMore> partitionAction = partitionActionsOfTable.get(partitionValues); if (partitionAction == null) { switch (tableSource) { case PRE_EXISTING_TABLE: partitionNamesToQuery.add(partitionName); break; case CREATED_IN_THIS_TRANSACTION: resultBuilder.put(partitionName, Optional.empty()); break; default: throw new UnsupportedOperationException("unknown table source"); } } else { resultBuilder.put(partitionName, getPartitionFromPartitionAction(partitionAction)); } } Map<String, Optional<Partition>> delegateResult = delegate.getPartitionsByNames(databaseName, tableName, partitionNamesToQuery.build()); resultBuilder.putAll(delegateResult); return resultBuilder.build(); }
private void validateMetadata(ExtendedHiveMetastore hiveMetastore) { assertEquals(hiveMetastore.getDatabase("database"), Optional.of(DATABASE)); assertEquals(hiveMetastore.getAllDatabases(), ImmutableList.of("database")); assertEquals(hiveMetastore.getTable("database", "table"), Optional.of(TABLE)); assertEquals(hiveMetastore.getSupportedColumnStatistics(createVarcharType(123)), ImmutableSet.of(MIN_VALUE, MAX_VALUE)); assertEquals(hiveMetastore.getTableStatistics("database", "table"), PARTITION_STATISTICS); assertEquals(hiveMetastore.getPartitionStatistics("database", "table", ImmutableSet.of("value")), ImmutableMap.of("value", PARTITION_STATISTICS)); assertEquals(hiveMetastore.getAllTables("database"), Optional.of(ImmutableList.of("table"))); assertEquals(hiveMetastore.getAllViews("database"), Optional.empty()); assertEquals(hiveMetastore.getPartition("database", "table", ImmutableList.of("value")), Optional.of(PARTITION)); assertEquals(hiveMetastore.getPartitionNames("database", "table"), Optional.of(ImmutableList.of("value"))); assertEquals(hiveMetastore.getPartitionNamesByParts("database", "table", ImmutableList.of("value")), Optional.of(ImmutableList.of("value"))); assertEquals(hiveMetastore.getPartitionsByNames("database", "table", ImmutableList.of("value")), ImmutableMap.of("value", Optional.of(PARTITION))); assertEquals(hiveMetastore.getRoles("user"), ImmutableSet.of("role1", "role2")); assertEquals(hiveMetastore.getDatabasePrivileges("user", "database"), ImmutableSet.of(PRIVILEGE_INFO)); assertEquals(hiveMetastore.getTablePrivileges("user", "database", "table"), ImmutableSet.of(PRIVILEGE_INFO)); }
.orElse(ImmutableList.of()); for (List<String> partitionNameBatch : Iterables.partition(partitionNames, 10)) { Collection<Optional<Partition>> partitions = delegate.getPartitionsByNames(schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionNameBatch).values(); partitions.stream() .filter(Optional::isPresent)
private void eraseStatistics(SchemaTableName schemaTableName) { ExtendedHiveMetastore metastoreClient = getMetastoreClient(); metastoreClient.updateTableStatistics(schemaTableName.getSchemaName(), schemaTableName.getTableName(), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); Table table = metastoreClient.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(schemaTableName)); List<String> partitionColumns = table.getPartitionColumns().stream() .map(Column::getName) .collect(toImmutableList()); if (!table.getPartitionColumns().isEmpty()) { List<String> partitionNames = metastoreClient.getPartitionNames(schemaTableName.getSchemaName(), schemaTableName.getTableName()) .orElse(ImmutableList.of()); List<Partition> partitions = metastoreClient .getPartitionsByNames(schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionNames) .entrySet() .stream() .map(Map.Entry::getValue) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); for (Partition partition : partitions) { metastoreClient.updatePartitionStatistics( schemaTableName.getSchemaName(), schemaTableName.getTableName(), makePartName(partitionColumns, partition.getValues()), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); } } }
Map<String, Optional<Partition>> partitions = getMetastoreClient().getPartitionsByNames(tableName.getSchemaName(), tableName.getTableName(), partitionNames); assertEquals(partitions.size(), partitionNames.size()); for (String partitionName : partitionNames) {
private void validateMetadata(ExtendedHiveMetastore hiveMetastore) { assertEquals(hiveMetastore.getDatabase("database"), Optional.of(DATABASE)); assertEquals(hiveMetastore.getAllDatabases(), ImmutableList.of("database")); assertEquals(hiveMetastore.getTable("database", "table"), Optional.of(TABLE)); assertEquals(hiveMetastore.getSupportedColumnStatistics(createVarcharType(123)), ImmutableSet.of(MIN_VALUE, MAX_VALUE)); assertEquals(hiveMetastore.getTableStatistics("database", "table"), PARTITION_STATISTICS); assertEquals(hiveMetastore.getPartitionStatistics("database", "table", ImmutableSet.of("value")), ImmutableMap.of("value", PARTITION_STATISTICS)); assertEquals(hiveMetastore.getAllTables("database"), Optional.of(ImmutableList.of("table"))); assertEquals(hiveMetastore.getAllViews("database"), Optional.empty()); assertEquals(hiveMetastore.getPartition("database", "table", ImmutableList.of("value")), Optional.of(PARTITION)); assertEquals(hiveMetastore.getPartitionNames("database", "table"), Optional.of(ImmutableList.of("value"))); assertEquals(hiveMetastore.getPartitionNamesByParts("database", "table", ImmutableList.of("value")), Optional.of(ImmutableList.of("value"))); assertEquals(hiveMetastore.getPartitionsByNames("database", "table", ImmutableList.of("value")), ImmutableMap.of("value", Optional.of(PARTITION))); assertEquals(hiveMetastore.getRoles("user"), ImmutableSet.of("role1", "role2")); assertEquals(hiveMetastore.getDatabasePrivileges("user", "database"), ImmutableSet.of(PRIVILEGE_INFO)); assertEquals(hiveMetastore.getTablePrivileges("user", "database", "table"), ImmutableSet.of(PRIVILEGE_INFO)); }
private void eraseStatistics(SchemaTableName schemaTableName) { ExtendedHiveMetastore metastoreClient = getMetastoreClient(); metastoreClient.updateTableStatistics(schemaTableName.getSchemaName(), schemaTableName.getTableName(), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); Table table = metastoreClient.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(schemaTableName)); List<String> partitionColumns = table.getPartitionColumns().stream() .map(Column::getName) .collect(toImmutableList()); if (!table.getPartitionColumns().isEmpty()) { List<String> partitionNames = metastoreClient.getPartitionNames(schemaTableName.getSchemaName(), schemaTableName.getTableName()) .orElse(ImmutableList.of()); List<Partition> partitions = metastoreClient .getPartitionsByNames(schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionNames) .entrySet() .stream() .map(Map.Entry::getValue) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); for (Partition partition : partitions) { metastoreClient.updatePartitionStatistics( schemaTableName.getSchemaName(), schemaTableName.getTableName(), makePartName(partitionColumns, partition.getValues()), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); } } }
Map<String, Optional<Partition>> partitions = getMetastoreClient().getPartitionsByNames(tableName.getSchemaName(), tableName.getTableName(), partitionNames); assertEquals(partitions.size(), partitionNames.size()); for (String partitionName : partitionNames) {