private static Optional<String> getPrestoQueryId(Partition partition) { return Optional.ofNullable(partition.getParameters().get(PRESTO_QUERY_ID_NAME)); }
public static ProtectMode getProtectMode(Partition partition) { return getProtectMode(partition.getParameters()); }
@Override public Map<String, PartitionStatistics> getPartitionStatistics(String databaseName, String tableName, Set<String> partitionNames) { ImmutableMap.Builder<String, PartitionStatistics> result = ImmutableMap.builder(); getPartitionsByNames(databaseName, tableName, ImmutableList.copyOf(partitionNames)).forEach((partitionName, optionalPartition) -> { Partition partition = optionalPartition.orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), toPartitionValues(partitionName))); PartitionStatistics partitionStatistics = new PartitionStatistics(getHiveBasicStatistics(partition.getParameters()), ImmutableMap.of()); result.put(partitionName, partitionStatistics); }); return result.build(); }
@Test public void testPartitionNullParameters() { testPartition.setParameters(null); assertNotNull(GlueToPrestoConverter.convertPartition(testPartition).getParameters()); }
public static void checkPartitionIsWritable(String partitionName, Partition partition) { checkWritable( new SchemaTableName(partition.getDatabaseName(), partition.getTableName()), Optional.of(partitionName), getProtectMode(partition), partition.getParameters(), partition.getStorage()); }
verifyOnline(tableName, Optional.of(partName), getProtectMode(partition), partition.getParameters()); String partitionNotReadable = partition.getParameters().get(OBJECT_NOT_READABLE); if (!isNullOrEmpty(partitionNotReadable)) { throw new HiveNotReadableException(tableName, Optional.of(partName), partitionNotReadable);
private Builder(Partition partition) { this.storageBuilder = Storage.builder(partition.getStorage()); this.databaseName = partition.getDatabaseName(); this.tableName = partition.getTableName(); this.values = partition.getValues(); this.columns = partition.getColumns(); this.parameters = partition.getParameters(); }
public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStatistics) { Partition partition = partitionWithStatistics.getPartition(); PartitionStatistics statistics = partitionWithStatistics.getStatistics(); this.columns = partition.getColumns(); this.parameters = updateStatisticsParameters(partition.getParameters(), statistics.getBasicStatistics()); StorageFormat tableFormat = partition.getStorage().getStorageFormat(); storageFormat = Arrays.stream(HiveStorageFormat.values()) .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) .findFirst(); if (table.getTableType().equals(TableType.EXTERNAL_TABLE.name())) { externalLocation = Optional.of(partition.getStorage().getLocation()); } else { externalLocation = Optional.empty(); } bucketProperty = partition.getStorage().getBucketProperty(); serdeParameters = partition.getStorage().getSerdeParameters(); columnStatistics = ImmutableMap.copyOf(statistics.getColumnStatistics()); }
@Test public void testConvertPartition() { PartitionInput partitionInput = GlueInputConverter.convertPartition(testPartition); assertEquals(partitionInput.getParameters(), testPartition.getParameters()); assertStorage(partitionInput.getStorageDescriptor(), testPartition.getStorage()); assertEquals(partitionInput.getValues(), testPartition.getValues()); }
public static PartitionInput convertPartition(Partition partition) { PartitionInput input = new PartitionInput(); input.setValues(partition.getValues()); input.setStorageDescriptor(convertStorage(partition.getStorage(), partition.getColumns())); input.setParameters(partition.getParameters()); return input; }
@Test public void testConvertPartition() { com.facebook.presto.hive.metastore.Partition prestoPartition = GlueToPrestoConverter.convertPartition(testPartition); assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); assertEquals(prestoPartition.getValues(), testPartition.getValues()); assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); }
@Override public void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) { PartitionStatistics currentStatistics = getPartitionStatistics(databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName); if (currentStatistics == null) { throw new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + partitionName); } PartitionStatistics updatedStatistics = update.apply(currentStatistics); if (!updatedStatistics.getColumnStatistics().isEmpty()) { throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics"); } List<String> partitionValues = toPartitionValues(partitionName); Partition partition = getPartition(databaseName, tableName, partitionValues) .orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues)); try { PartitionInput partitionInput = GlueInputConverter.convertPartition(partition); partitionInput.setParameters(updateStatisticsParameters(partition.getParameters(), updatedStatistics.getBasicStatistics())); glueClient.updatePartition(new UpdatePartitionRequest() .withDatabaseName(databaseName) .withTableName(tableName) .withPartitionValueList(partition.getValues()) .withPartitionInput(partitionInput)); } catch (EntityNotFoundException e) { throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues); } catch (AmazonServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition) { org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition(); result.setDbName(partition.getDatabaseName()); result.setTableName(partition.getTableName()); result.setValues(partition.getValues()); result.setSd(makeStorageDescriptor(partition.getTableName(), partition.getColumns(), partition.getStorage())); result.setParameters(partition.getParameters()); return result; }
for (String partitionName : partitionNames) { Partition partition = partitions.get(partitionName).get(); assertEquals(partition.getParameters().get(PRESTO_VERSION_NAME), TEST_SERVER_VERSION); assertEquals(partition.getParameters().get(PRESTO_QUERY_ID_NAME), queryId);
@Test public void testPartitionNullParameters() { testPartition.setParameters(null); assertNotNull(GlueToPrestoConverter.convertPartition(testPartition).getParameters()); }
@Test public void testConvertPartition() { PartitionInput partitionInput = GlueInputConverter.convertPartition(testPartition); assertEquals(partitionInput.getParameters(), testPartition.getParameters()); assertStorage(partitionInput.getStorageDescriptor(), testPartition.getStorage()); assertEquals(partitionInput.getValues(), testPartition.getValues()); }
@Test public void testConvertPartition() { com.facebook.presto.hive.metastore.Partition prestoPartition = GlueToPrestoConverter.convertPartition(testPartition); assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); assertEquals(prestoPartition.getValues(), testPartition.getValues()); assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); }
for (String partitionName : partitionNames) { Partition partition = partitions.get(partitionName).get(); assertEquals(partition.getParameters().get(PRESTO_VERSION_NAME), TEST_SERVER_VERSION); assertEquals(partition.getParameters().get(PRESTO_QUERY_ID_NAME), queryId);