public Partition getAugmentedPartitionForInTransactionRead() { // This method augments the location field of the partition to the staging location. // This way, if the partition is accessed in an ongoing transaction, staged data // can be found and accessed. Partition partition = this.partition; String currentLocation = this.currentLocation.toString(); if (!currentLocation.equals(partition.getStorage().getLocation())) { partition = Partition.builder(partition) .withStorage(storage -> storage.setLocation(currentLocation)) .build(); } return partition; }
public static Partition convertPartition(com.amazonaws.services.glue.model.Partition gluePartition) { requireNonNull(gluePartition.getStorageDescriptor(), "Partition StorageDescriptor is null"); StorageDescriptor sd = gluePartition.getStorageDescriptor(); Partition.Builder partitionBuilder = Partition.builder() .setDatabaseName(gluePartition.getDatabaseName()) .setTableName(gluePartition.getTableName()) .setValues(gluePartition.getValues()) .setColumns(sd.getColumns().stream() .map(GlueToPrestoConverter::convertColumn) .collect(toList())) .setParameters(firstNonNull(gluePartition.getParameters(), ImmutableMap.of())); setStorageBuilder(sd, partitionBuilder.getStorageBuilder()); return partitionBuilder.build(); }
public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition) { StorageDescriptor storageDescriptor = partition.getSd(); if (storageDescriptor == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Partition does not contain a storage descriptor: " + partition); } Partition.Builder partitionBuilder = Partition.builder() .setDatabaseName(partition.getDbName()) .setTableName(partition.getTableName()) .setValues(partition.getValues()) .setColumns(storageDescriptor.getCols().stream() .map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema) .collect(toList())) .setParameters(partition.getParameters()); fromMetastoreApiStorageDescriptor(storageDescriptor, partitionBuilder.getStorageBuilder(), format("%s.%s", partition.getTableName(), partition.getValues())); return partitionBuilder.build(); }
public static com.facebook.presto.hive.metastore.Partition getPrestoTestPartition(String dbName, String tblName, List<String> values) { return com.facebook.presto.hive.metastore.Partition.builder() .setDatabaseName(dbName) .setTableName(tblName) .setValues(values) .setColumns(ImmutableList.of(getPrestoTestColumn())) .setParameters(ImmutableMap.of()) .withStorage(STORAGE_CONSUMER).build(); }
private Partition buildPartitionObject(ConnectorSession session, Table table, PartitionUpdate partitionUpdate) { return Partition.builder() .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) .setValues(extractPartitionValues(partitionUpdate.getName())) .setParameters(ImmutableMap.<String, String>builder() .put(PRESTO_VERSION_NAME, prestoVersion) .put(PRESTO_QUERY_ID_NAME, session.getQueryId()) .build()) .withStorage(storage -> storage .setStorageFormat(isRespectTableFormat(session) ? table.getStorage().getStorageFormat() : fromHiveStorageFormat(HiveSessionProperties.getHiveStorageFormat(session))) .setLocation(partitionUpdate.getTargetPath().toString()) .setBucketProperty(table.getStorage().getBucketProperty()) .setSerdeParameters(table.getStorage().getSerdeParameters())) .build(); }
@Override public void triggerConflict(ConnectorSession session, SchemaTableName tableName, ConnectorInsertTableHandle insertTableHandle, List<PartitionUpdate> partitionUpdates) { // This method bypasses transaction interface because this method is inherently hacky and doesn't work well with the transaction abstraction. // Additionally, this method is not part of a test. Its purpose is to set up an environment for another test. ExtendedHiveMetastore metastoreClient = getMetastoreClient(); Optional<Partition> partition = metastoreClient.getPartition(tableName.getSchemaName(), tableName.getTableName(), copyPartitionFrom); conflictPartition = Partition.builder(partition.get()) .setValues(toPartitionValues(partitionNameToConflict)) .build(); metastoreClient.addPartitions( tableName.getSchemaName(), tableName.getTableName(), ImmutableList.of(new PartitionWithStatistics(conflictPartition, partitionNameToConflict, PartitionStatistics.empty()))); }
Partition modifiedPartition = Partition.builder(partition) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(DWRF)) modifiedPartition = Partition.builder(partition) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(TEXTFILE)) modifiedPartition = Partition.builder(partition) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(TEXTFILE))
protected Partition createDummyPartition(Table table, String partitionName) { return Partition.builder() .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) .setValues(toPartitionValues(partitionName)) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(HiveStorageFormat.ORC)) .setLocation(partitionTargetPath(new SchemaTableName(table.getDatabaseName(), table.getTableName()), partitionName))) .setParameters(ImmutableMap.of( PRESTO_VERSION_NAME, "testversion", PRESTO_QUERY_ID_NAME, "20180101_123456_00001_x1y2z")) .build(); }
public static com.facebook.presto.hive.metastore.Partition getPrestoTestPartition(String dbName, String tblName, List<String> values) { return com.facebook.presto.hive.metastore.Partition.builder() .setDatabaseName(dbName) .setTableName(tblName) .setValues(values) .setColumns(ImmutableList.of(getPrestoTestColumn())) .setParameters(ImmutableMap.of()) .withStorage(STORAGE_CONSUMER).build(); }
@Override public void triggerConflict(ConnectorSession session, SchemaTableName tableName, ConnectorInsertTableHandle insertTableHandle, List<PartitionUpdate> partitionUpdates) { // This method bypasses transaction interface because this method is inherently hacky and doesn't work well with the transaction abstraction. // Additionally, this method is not part of a test. Its purpose is to set up an environment for another test. ExtendedHiveMetastore metastoreClient = getMetastoreClient(); Optional<Partition> partition = metastoreClient.getPartition(tableName.getSchemaName(), tableName.getTableName(), copyPartitionFrom); conflictPartition = Partition.builder(partition.get()) .setValues(toPartitionValues(partitionNameToConflict)) .build(); metastoreClient.addPartitions( tableName.getSchemaName(), tableName.getTableName(), ImmutableList.of(new PartitionWithStatistics(conflictPartition, partitionNameToConflict, PartitionStatistics.empty()))); }
Partition modifiedPartition = Partition.builder(partition) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(DWRF)) modifiedPartition = Partition.builder(partition) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(TEXTFILE)) modifiedPartition = Partition.builder(partition) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(TEXTFILE))
protected Partition createDummyPartition(Table table, String partitionName) { return Partition.builder() .setDatabaseName(table.getDatabaseName()) .setTableName(table.getTableName()) .setColumns(table.getDataColumns()) .setValues(toPartitionValues(partitionName)) .withStorage(storage -> storage .setStorageFormat(fromHiveStorageFormat(HiveStorageFormat.ORC)) .setLocation(partitionTargetPath(new SchemaTableName(table.getDatabaseName(), table.getTableName()), partitionName))) .setParameters(ImmutableMap.of( PRESTO_VERSION_NAME, "testversion", PRESTO_QUERY_ID_NAME, "20180101_123456_00001_x1y2z")) .build(); }