public synchronized void truncateUnpartitionedTable(ConnectorSession session, String databaseName, String tableName) { checkReadable(); Optional<Table> table = getTable(databaseName, tableName); SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(schemaTableName); } if (!table.get().getTableType().equals(MANAGED_TABLE.toString())) { throw new PrestoException(NOT_SUPPORTED, "Cannot delete from non-managed Hive table"); } if (!table.get().getPartitionColumns().isEmpty()) { throw new IllegalArgumentException("Table is partitioned"); } Path path = new Path(table.get().getStorage().getLocation()); HdfsContext context = new HdfsContext(session, databaseName, tableName); setExclusive((delegate, hdfsEnvironment) -> { RecursiveDeleteResult recursiveDeleteResult = recursiveDeleteFiles(hdfsEnvironment, context, path, ImmutableList.of(""), false); if (!recursiveDeleteResult.getNotDeletedEligibleItems().isEmpty()) { throw new PrestoException(HIVE_FILESYSTEM_ERROR, format( "Error deleting from unpartitioned table %s. These items can not be deleted: %s", schemaTableName, recursiveDeleteResult.getNotDeletedEligibleItems())); } }); }
@Override public synchronized Optional<List<String>> getAllViews(String databaseName) { Optional<List<String>> tables = getAllTables(databaseName); if (!tables.isPresent()) { return Optional.empty(); } List<String> views = tables.get().stream() .map(tableName -> getTable(databaseName, tableName)) .filter(Optional::isPresent) .map(Optional::get) .filter(table -> table.getTableType().equals(VIRTUAL_VIEW.name())) .map(Table::getTableName) .collect(toList()); return Optional.of(ImmutableList.copyOf(views)); }
TableType tableType = TableType.valueOf(table.getTableType()); checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(tableType), "Invalid table type: %s", tableType); verifiedPartition(table, partition); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); Path schemaPath = new Path(partitionMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); if (metadataFileSystem.exists(schemaPath)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Partition already exists");
public ClassicTableTypeMapping () { hiveToClientMap.put(TableType.MANAGED_TABLE.name(), ClassicTableTypes.TABLE.name()); hiveToClientMap.put(TableType.EXTERNAL_TABLE.name(), ClassicTableTypes.TABLE.name()); hiveToClientMap.put(TableType.VIRTUAL_VIEW.name(), ClassicTableTypes.VIEW.name()); hiveToClientMap.put(TableType.MATERIALIZED_VIEW.toString(), ClassicTableTypes.MATERIALIZED_VIEW.toString()); clientToHiveMap.putAll(ClassicTableTypes.TABLE.name(), Arrays.asList( TableType.MANAGED_TABLE.name(), TableType.EXTERNAL_TABLE.name())); clientToHiveMap.put(ClassicTableTypes.VIEW.name(), TableType.VIRTUAL_VIEW.name()); clientToHiveMap.put(ClassicTableTypes.MATERIALIZED_VIEW.toString(), TableType.MATERIALIZED_VIEW.toString()); }
.build(); Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty()); .setTableName(viewName.getTableName()) .setOwner(session.getUser()) .setTableType(TableType.VIRTUAL_VIEW.name()) .setDataColumns(ImmutableList.of(dummyColumn)) .setPartitionColumns(ImmutableList.of()) .setParameters(properties) .setViewOriginalText(Optional.of(encodeViewData(viewData))) .setViewExpandedText(Optional.of("/* Presto View */"));
private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) { columns.add(metadataGetter.apply(columnHandle)); if (table.get().getTableType().equals(EXTERNAL_TABLE.name())) { properties.put(EXTERNAL_LOCATION_PROPERTY, table.get().getStorage().getLocation());
if (table.getTableType().equals(MANAGED_TABLE.name())) { String targetLocation = table.getStorage().getLocation(); checkArgument(!targetLocation.isEmpty(), "target location is empty"); Optional<Path> currentPath = tableAndMore.getCurrentLocation(); Path targetPath = new Path(targetLocation); if (table.getPartitionColumns().isEmpty() && currentPath.isPresent()) { if (targetPath.equals(currentPath.get())) { context, hdfsEnvironment, currentPath.get(), targetPath, () -> cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, true))); if (currentPath.isPresent() && currentPath.get().equals(targetPath)) {
@Override public Optional<List<String>> getAllViews(String databaseName) { try { List<String> views = new ArrayList<>(); String nextToken = null; do { GetTablesResult result = glueClient.getTables(new GetTablesRequest() .withDatabaseName(databaseName) .withNextToken(nextToken)); result.getTableList().stream() .filter(table -> VIRTUAL_VIEW.name().equals(table.getTableType())) .forEach(table -> views.add(table.getName())); nextToken = result.getNextToken(); } while (nextToken != null); return Optional.of(views); } catch (EntityNotFoundException e) { // database does not exist return Optional.empty(); } catch (AmazonServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }
public static com.facebook.presto.hive.metastore.Table getPrestoTestTable(String dbName) { return com.facebook.presto.hive.metastore.Table.builder() .setDatabaseName(dbName) .setTableName("test-tbl" + generateRandom()) .setOwner("owner") .setParameters(ImmutableMap.of()) .setTableType(TableType.EXTERNAL_TABLE.name()) .setDataColumns(ImmutableList.of(getPrestoTestColumn())) .setPartitionColumns(ImmutableList.of(getPrestoTestColumn())) .setViewOriginalText(Optional.of("originalText")) .setViewExpandedText(Optional.of("expandedText")) .withStorage(STORAGE_CONSUMER).build(); }
@Override public Optional<Table> getTable(String databaseName, String tableName) { try { return retry() .stopOn(NoSuchObjectException.class, HiveViewNotSupportedException.class) .stopOnIllegalExceptions() .run("getTable", stats.getGetTable().wrap(() -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { Table table = client.getTable(databaseName, tableName); if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name()) && !isPrestoView(table)) { throw new HiveViewNotSupportedException(new SchemaTableName(databaseName, tableName)); } return Optional.of(table); } })); } catch (NoSuchObjectException e) { return Optional.empty(); } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } catch (Exception e) { throw propagate(e); } }
this.dataColumns = ImmutableList.copyOf(requireNonNull(dataColumns, "dataColumns is null")); this.partitionColumns = ImmutableList.copyOf(requireNonNull(partitionColumns, "partitionColumns is null")); this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); this.serdeParameters = requireNonNull(serdeParameters, "serdeParameters is null"); this.externalLocation = requireNonNull(externalLocation, "externalLocation is null"); if (tableType.equals(TableType.EXTERNAL_TABLE.name())) { checkArgument(externalLocation.isPresent(), "External location is required for external tables"); checkArgument(!externalLocation.isPresent(), "External location is only allowed for external tables");
.setTableName(tableName) .setOwner(tableOwner) .setTableType(TableType.MANAGED_TABLE.name()) .setParameters(ImmutableMap.of( PRESTO_VERSION_NAME, TEST_SERVER_VERSION, .setLocation(targetPath.toString()) .setStorageFormat(StorageFormat.create(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat())) .setBucketProperty(bucketProperty) transaction.getMetastore(schemaName).createTable(session, tableBuilder.build(), principalPrivileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS); assertEquals(targetDirectoryList, ImmutableList.of());
private static Table table( List<Column> partitionColumns, Optional<HiveBucketProperty> bucketProperty) { Table.Builder tableBuilder = Table.builder(); tableBuilder.getStorageBuilder() .setStorageFormat( StorageFormat.create( "com.facebook.hive.orc.OrcSerde", "org.apache.hadoop.hive.ql.io.RCFileInputFormat", "org.apache.hadoop.hive.ql.io.RCFileInputFormat")) .setLocation("hdfs://VOL1:9000/db_name/table_name") .setSkewed(false) .setBucketProperty(bucketProperty); return tableBuilder .setDatabaseName("test_dbname") .setOwner("testOwner") .setTableName("test_table") .setTableType(TableType.MANAGED_TABLE.toString()) .setDataColumns(ImmutableList.of(new Column("col1", HIVE_STRING, Optional.empty()))) .setParameters(ImmutableMap.of()) .setPartitionColumns(partitionColumns) .build(); }
@Override public Table getTable(String dbName, String tableName) throws TException { accessCount.incrementAndGet(); if (throwException) { throw new RuntimeException(); } if (!dbName.equals(TEST_DATABASE) || !tableName.equals(TEST_TABLE)) { throw new NoSuchObjectException(); } return new Table( TEST_TABLE, TEST_DATABASE, "", 0, 0, 0, DEFAULT_STORAGE_DESCRIPTOR, ImmutableList.of(new FieldSchema("key", "string", null)), null, "", "", TableType.MANAGED_TABLE.name()); }
public static void checkTableIsWritable(Table table, boolean writesToNonManagedTablesEnabled) { if (!writesToNonManagedTablesEnabled && !table.getTableType().equals(MANAGED_TABLE.toString())) { throw new PrestoException(NOT_SUPPORTED, "Cannot write to non-managed Hive table"); } checkWritable( new SchemaTableName(table.getDatabaseName(), table.getTableName()), Optional.empty(), getProtectMode(table), table.getParameters(), table.getStorage()); }
public static Table getGlueTestTable(String dbName) { return new Table() .withDatabaseName(dbName) .withName("test-tbl" + generateRandom()) .withOwner("owner") .withParameters(ImmutableMap.of()) .withPartitionKeys(ImmutableList.of(getGlueTestColumn())) .withStorageDescriptor(getGlueTestStorageDescriptor()) .withTableType(TableType.EXTERNAL_TABLE.name()) .withViewOriginalText("originalText") .withViewExpandedText("expandedText"); }
@Override public synchronized void createTable(Table table) TableType tableType = TableType.valueOf(table.getTableType()); checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW).contains(tableType), "Invalid table type: %s", tableType); checkArgument(table.getSd().getLocation() == null, "Storage location for view must be null"); File directory = new File(new Path(table.getSd().getLocation()).toUri()); checkArgument(directory.exists(), "Table directory does not exist"); if (tableType == MANAGED_TABLE) {
ImmutableList.Builder<Column> columns = ImmutableList.builder(); for (HiveColumnHandle columnHandle : columnHandles) { String name = columnHandle.getName(); .setTableName(tableName) .setOwner(tableOwner) .setTableType((external ? EXTERNAL_TABLE : MANAGED_TABLE).name()) .setDataColumns(columns.build()) .setPartitionColumns(partitionColumns) .setStorageFormat(fromHiveStorageFormat(hiveStorageFormat)) .setBucketProperty(bucketProperty) .setLocation(targetPath.toString());