private Optional<ConnectorTableMetadata> getTableMetadata(SchemaTableName tableName) { ElasticsearchTableDescription table = client.getTable(tableName.getSchemaName(), tableName.getTableName()); if (table == null) { return Optional.empty(); } return Optional.of(new ConnectorTableMetadata(tableName, client.getColumnMetadata(table))); }
private static List<AccumuloColumnHandle> getColumnHandles(ConnectorTableMetadata meta, String rowIdColumn) Map<String, Pair<String, String>> mapping = AccumuloTableProperties.getColumnMapping(meta.getProperties()).orElse(autoGenerateMapping(meta.getColumns(), AccumuloTableProperties.getLocalityGroups(meta.getProperties()))); Optional<List<String>> indexedColumns = AccumuloTableProperties.getIndexColumns(meta.getProperties()); ImmutableList.Builder<AccumuloColumnHandle> cBuilder = ImmutableList.builder(); for (int ordinal = 0; ordinal < meta.getColumns().size(); ++ordinal) { ColumnMetadata cm = meta.getColumns().get(ordinal); if (cm.getName().equalsIgnoreCase(rowIdColumn)) { cBuilder.add( new AccumuloColumnHandle( rowIdColumn, Optional.empty(), Optional.empty(), cm.getType(), ordinal, "Accumulo row ID", if (!mapping.containsKey(cm.getName())) { throw new InvalidParameterException(format("Misconfigured mapping for presto column %s", cm.getName())); cBuilder.add( new AccumuloColumnHandle( cm.getName(), return cBuilder.build();
Optional<ConnectorNewTableLayout> layout) PartitionDesign design = KuduTableProperties.getPartitionDesign(tableMetadata.getProperties()); boolean generateUUID = !design.hasPartitions(); ConnectorTableMetadata finalTableMetadata = tableMetadata; if (generateUUID) { String rowId = KuduColumnHandle.ROW_ID; List<ColumnMetadata> copy = new ArrayList<>(tableMetadata.getColumns()); Map<String, Object> columnProperties = new HashMap<>(); columnProperties.put(KuduTableProperties.PRIMARY_KEY, true); copy.add(0, new ColumnMetadata(rowId, VarcharType.VARCHAR, "key=true", null, true, columnProperties)); List<ColumnMetadata> finalColumns = ImmutableList.copyOf(copy); Map<String, Object> propsCopy = new HashMap<>(tableMetadata.getProperties()); propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_COLUMNS, ImmutableList.of(rowId)); propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_BUCKETS, 2); Map<String, Object> finalProperties = ImmutableMap.copyOf(propsCopy); finalTableMetadata = new ConnectorTableMetadata(tableMetadata.getTable(), finalColumns, finalProperties, tableMetadata.getComment()); .map(TypeHelper::fromKuduColumn).collect(toImmutableList()); List<Type> columnOriginalTypes = finalTableMetadata.getColumns().stream() .map(ColumnMetadata::getType).collect(toImmutableList()); finalTableMetadata.getTable(), columnOriginalTypes, columnTypes,
public MemoryTableHandle( String connectorId, Long tableId, ConnectorTableMetadata tableMetadata) { this(connectorId, tableMetadata.getTable().getSchemaName(), tableMetadata.getTable().getTableName(), tableId, MemoryColumnHandle.extractColumnHandles(tableMetadata.getColumns())); }
@Override public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); columns.addAll(tableMetadata.getColumns()); columns.add(column); tables.put(tableName, new ConnectorTableMetadata(tableName, columns.build(), tableMetadata.getProperties(), tableMetadata.getComment())); }
/** * Gets the row ID based on a table properties or the first column name. * * @param meta ConnectorTableMetadata * @return Lowercase Presto column name mapped to the Accumulo row ID */ private static String getRowIdColumn(ConnectorTableMetadata meta) { Optional<String> rowIdColumn = AccumuloTableProperties.getRowId(meta.getProperties()); return rowIdColumn.orElse(meta.getColumns().get(0).getName()).toLowerCase(Locale.ENGLISH); }
if (!viewDefinition.isPresent()) { if (metadata.getTableHandle(session, objectName).isPresent()) { throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a view", objectName); Query query = parseView(viewDefinition.get().getOriginalSql(), objectName, node); String sql = formatSql(new CreateView(createQualifiedName(objectName), query, false), Optional.of(parameters)).trim(); return singleValueQuery("Create View", sql); Map<String, PropertyMetadata<?>> allColumnProperties = metadata.getColumnPropertyManager().getAllProperties().get(tableHandle.get().getConnectorId()); List<TableElement> columns = connectorTableMetadata.getColumns().stream() .filter(column -> !column.isHidden()) .map(column -> { List<Property> propertyNodes = buildProperties(objectName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties); return new ColumnDefinition(new Identifier(column.getName()), column.getType().getDisplayName(), propertyNodes, Optional.ofNullable(column.getComment())); }) .collect(toImmutableList()); Map<String, Object> properties = connectorTableMetadata.getProperties(); Map<String, PropertyMetadata<?>> allTableProperties = metadata.getTablePropertyManager().getAllProperties().get(tableHandle.get().getConnectorId()); List<Property> propertyNodes = buildProperties(objectName, Optional.empty(), INVALID_TABLE_PROPERTY, properties, allTableProperties); false, propertyNodes, connectorTableMetadata.getComment()); return singleValueQuery("Create Table", formatSql(createTable, Optional.of(parameters)).trim());
if (tableHandle.isPresent()) { if (!statement.isNotExists()) { throw new SemanticException(TABLE_ALREADY_EXISTS, statement, "Table '%s' already exists", tableName); .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName())); Map<String, Object> inheritedProperties = ImmutableMap.of(); boolean includingProperties = false; for (TableElement element : statement.getElements()) { parameters); columns.put(name, new ColumnMetadata(name, type, column.getComment().orElse(null), null, false, columnProperties)); inheritedProperties = likeTableMetadata.getMetadata().getProperties(); .filter(column -> !column.isHidden()) .forEach(column -> { if (columns.containsKey(column.getName().toLowerCase(Locale.ENGLISH))) { throw new SemanticException(DUPLICATE_COLUMN_NAME, element, "Column name '%s' specified more than once", column.getName()); ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName.asSchemaTableName(), ImmutableList.copyOf(columns.values()), finalProperties, statement.getComment()); try { metadata.createTable(session, tableName.getCatalogName(), tableMetadata, statement.isNotExists());
@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata tableMetadata) { validatePartitionColumns(tableMetadata); validateBucketColumns(tableMetadata); Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return Optional.empty(); } if (!bucketProperty.get().getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) { throw new PrestoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled"); } List<String> bucketedBy = bucketProperty.get().getBucketedBy(); Map<String, HiveType> hiveTypeMap = tableMetadata.getColumns().stream() .collect(toMap(ColumnMetadata::getName, column -> toHiveType(typeTranslator, column.getType()))); return Optional.of(new ConnectorNewTableLayout( new HivePartitioningHandle( bucketProperty.get().getBucketCount(), bucketedBy.stream() .map(hiveTypeMap::get) .collect(toList()), OptionalInt.of(bucketProperty.get().getBucketCount())), bucketedBy)); }
private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) { columns.add(metadataGetter.apply(columnHandle)); ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder(); if (table.get().getTableType().equals(EXTERNAL_TABLE.name())) { properties.put(EXTERNAL_LOCATION_PROPERTY, table.get().getStorage().getLocation()); HiveStorageFormat format = extractHiveStorageFormat(table.get()); properties.put(STORAGE_FORMAT_PROPERTY, format); .collect(toList()); if (!partitionedBy.isEmpty()) { properties.put(PARTITIONED_BY_PROPERTY, partitionedBy); Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT)); return new ConnectorTableMetadata(tableName, columns.build(), properties.build(), comment);
ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, CREATE_TABLE_COLUMNS, createTableProperties(storageFormat)); ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty()); HdfsContext context = new HdfsContext(session, tableName.getSchemaName(), tableName.getTableName()); for (String filePath : listAllDataFiles(context, getStagingPathRoot(outputHandle))) { assertTrue(new Path(filePath).getName().startsWith(getFilePrefix(outputHandle))); metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of()); assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), CREATE_TABLE_COLUMNS); MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), CREATE_TABLE_DATA.getMaterializedRows()); Table table = getMetastoreClient().getTable(tableName.getSchemaName(), tableName.getTableName()).get(); assertEquals(table.getParameters().get(PRESTO_VERSION_NAME), TEST_SERVER_VERSION); assertEquals(table.getParameters().get(PRESTO_QUERY_ID_NAME), queryId);
@Test public void tableIsCreatedAfterCommits() { assertNoTables(); SchemaTableName schemaTableName = new SchemaTableName("default", "temp_table"); ConnectorOutputTableHandle table = metadata.beginCreateTable( SESSION, new ConnectorTableMetadata(schemaTableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty()); metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of()); List<SchemaTableName> tables = metadata.listTables(SESSION, Optional.empty()); assertTrue(tables.size() == 1, "Expected only one table"); assertTrue(tables.get(0).getTableName().equals("temp_table"), "Expected table with name 'temp_table'"); }
return Optional.empty(); return Optional.empty(); .map(HiveColumnHandle::getTypeSignature) .map(typeManager::getType) .collect(toImmutableList()); .map(column -> new ColumnMetadata( column.getName(), typeManager.getType(column.getTypeSignature()), column.getComment().orElse(null), column.isHidden())) .collect(toImmutableList()); .collect(toImmutableMap(identity(), partitionColumns::get)); public ConnectorTableMetadata getTableMetadata() return new ConnectorTableMetadata(tableName, partitionSystemTableColumns);
@Test public void testCreateSchema() { assertEquals(metadata.listSchemaNames(SESSION), ImmutableList.of("default")); metadata.createSchema(SESSION, "test", ImmutableMap.of()); assertEquals(metadata.listSchemaNames(SESSION), ImmutableList.of("default", "test")); assertEquals(metadata.listTables(SESSION, "test"), ImmutableList.of()); SchemaTableName tableName = new SchemaTableName("test", "first_table"); metadata.createTable( SESSION, new ConnectorTableMetadata( tableName, ImmutableList.of(), ImmutableMap.of()), false); assertEquals(metadata.listTables(SESSION, Optional.empty()), ImmutableList.of(tableName)); assertEquals(metadata.listTables(SESSION, Optional.of("test")), ImmutableList.of(tableName)); assertEquals(metadata.listTables(SESSION, Optional.of("default")), ImmutableList.of()); }
private void createTable(SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception List<ColumnMetadata> columns = ImmutableList.<ColumnMetadata>builder() .add(new ColumnMetadata("id", BIGINT)) .build(); ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, createTableProperties(storageFormat)); ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty()); metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of()); tableName.getTableName(), locationService.getTableWriteInfo(((HiveOutputTableHandle) outputHandle).getLocationHandle()).getTargetPath().toString()); assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
@Override public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) { requireNonNull(prefix, "prefix is null"); ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix)) { Optional<ConnectorTableMetadata> tableMetadata = getTableMetadata(tableName); // table can disappear during listing operation if (tableMetadata.isPresent()) { columns.put(tableName, tableMetadata.get().getColumns()); } } return columns.build(); }
private static void validateLocalityGroups(ConnectorTableMetadata meta) Optional<Map<String, Set<String>>> groups = AccumuloTableProperties.getLocalityGroups(meta.getProperties()); if (!groups.isPresent()) { return; for (Map.Entry<String, Set<String>> g : groups.get().entrySet()) { if (g.getValue().contains(rowIdColumn)) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Row ID column cannot be in a locality group"); for (ColumnMetadata column : meta.getColumns()) { if (g.getValue().contains(column.getName().toLowerCase(Locale.ENGLISH))) { ++matchingColumns;
private static void validateBucketColumns(ConnectorTableMetadata tableMetadata) { Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return; } Set<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet()); List<String> bucketedBy = bucketProperty.get().getBucketedBy(); if (!allColumns.containsAll(bucketedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Bucketing columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(bucketedBy), ImmutableSet.copyOf(allColumns)))); } List<String> sortedBy = bucketProperty.get().getSortedBy().stream() .map(SortingColumn::getColumnName) .collect(toImmutableList()); if (!allColumns.containsAll(sortedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Sorting columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(sortedBy), ImmutableSet.copyOf(allColumns)))); } }
SystemTable systemTable = tables.getSystemTable(session, tableName) .orElseThrow(() -> new PrestoException(NOT_FOUND, format("Table %s not found", tableName))); List<ColumnMetadata> tableColumns = systemTable.getTableMetadata().getColumns(); if (columnsByName.put(column.getName(), i) != null) { throw new PrestoException(GENERIC_INTERNAL_ERROR, "Duplicate column name: " + column.getName()); ImmutableList.Builder<Integer> userToSystemFieldIndex = ImmutableList.builder(); for (ColumnHandle column : columns) { String columnName = ((SystemColumnHandle) column).getColumnName(); userToSystemFieldIndex.add(index); ImmutableMap.Builder<Integer, Domain> newConstraints = ImmutableMap.builder(); for (Map.Entry<ColumnHandle, Domain> entry : constraint.getDomains().get().entrySet()) { String columnName = ((SystemColumnHandle) entry.getKey()).getColumnName(); newConstraints.put(columnsByName.get(columnName), entry.getValue()); TupleDomain<Integer> newContraint = withColumnDomains(newConstraints.build()); return new MappedPageSource(systemTable.pageSource(systemTransaction.getConnectorTransactionHandle(), session, newContraint), userToSystemFieldIndex.build()); return new RecordPageSource(new MappedRecordSet(toRecordSet(systemTransaction.getConnectorTransactionHandle(), systemTable, session, newContraint), userToSystemFieldIndex.build()));
@Test public void testRenameTable() { SchemaTableName tableName = new SchemaTableName("test_schema", "test_table_to_be_renamed"); metadata.createSchema(SESSION, "test_schema", ImmutableMap.of()); ConnectorOutputTableHandle table = metadata.beginCreateTable( SESSION, new ConnectorTableMetadata(tableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty()); metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of()); // rename table to schema which does not exist SchemaTableName invalidSchemaTableName = new SchemaTableName("test_schema_not_exist", "test_table_renamed"); ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, tableName); Throwable throwable = expectThrows(SchemaNotFoundException.class, () -> metadata.renameTable(SESSION, tableHandle, invalidSchemaTableName)); assertTrue(throwable.getMessage().equals("Schema test_schema_not_exist not found")); // rename table to same schema SchemaTableName sameSchemaTableName = new SchemaTableName("test_schema", "test_renamed"); metadata.renameTable(SESSION, metadata.getTableHandle(SESSION, tableName), sameSchemaTableName); assertEquals(metadata.listTables(SESSION, "test_schema"), ImmutableList.of(sameSchemaTableName)); // rename table to different schema metadata.createSchema(SESSION, "test_different_schema", ImmutableMap.of()); SchemaTableName differentSchemaTableName = new SchemaTableName("test_different_schema", "test_renamed"); metadata.renameTable(SESSION, metadata.getTableHandle(SESSION, sameSchemaTableName), differentSchemaTableName); assertEquals(metadata.listTables(SESSION, "test_schema"), ImmutableList.of()); assertEquals(metadata.listTables(SESSION, "test_different_schema"), ImmutableList.of(differentSchemaTableName)); }