Optional<ConnectorNewTableLayout> layout) PartitionDesign design = KuduTableProperties.getPartitionDesign(tableMetadata.getProperties()); boolean generateUUID = !design.hasPartitions(); ConnectorTableMetadata finalTableMetadata = tableMetadata; if (generateUUID) { String rowId = KuduColumnHandle.ROW_ID; List<ColumnMetadata> copy = new ArrayList<>(tableMetadata.getColumns()); Map<String, Object> columnProperties = new HashMap<>(); columnProperties.put(KuduTableProperties.PRIMARY_KEY, true); copy.add(0, new ColumnMetadata(rowId, VarcharType.VARCHAR, "key=true", null, true, columnProperties)); List<ColumnMetadata> finalColumns = ImmutableList.copyOf(copy); Map<String, Object> propsCopy = new HashMap<>(tableMetadata.getProperties()); propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_COLUMNS, ImmutableList.of(rowId)); propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_BUCKETS, 2); Map<String, Object> finalProperties = ImmutableMap.copyOf(propsCopy); .map(TypeHelper::fromKuduColumn).collect(toImmutableList()); List<Type> columnOriginalTypes = finalTableMetadata.getColumns().stream() .map(ColumnMetadata::getType).collect(toImmutableList());
private TableStatisticsMetadata getStatisticsCollectionMetadata(List<ColumnMetadata> columns, List<String> partitionedBy) { Set<ColumnStatisticMetadata> columnStatistics = columns.stream() .filter(column -> !partitionedBy.contains(column.getName())) .filter(column -> !column.isHidden()) .map(this::getColumnStatisticMetadata) .flatMap(List::stream) .collect(toImmutableSet()); return new TableStatisticsMetadata(columnStatistics, ImmutableSet.of(), partitionedBy); }
protected final List<Type> getColumnTypes(String tableName, String... columnNames) { checkState(session.getCatalog().isPresent(), "catalog not set"); checkState(session.getSchema().isPresent(), "schema not set"); // look up the table Metadata metadata = localQueryRunner.getMetadata(); QualifiedObjectName qualifiedTableName = new QualifiedObjectName(session.getCatalog().get(), session.getSchema().get(), tableName); TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName) .orElseThrow(() -> new IllegalArgumentException(format("Table %s does not exist", qualifiedTableName))); Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle); return Arrays.stream(columnNames) .map(allColumnHandles::get) .map(columnHandle -> metadata.getColumnMetadata(session, tableHandle, columnHandle).getType()) .collect(toImmutableList()); }
public ThriftColumnHandle(ColumnMetadata columnMetadata) { this(columnMetadata.getName(), columnMetadata.getType(), columnMetadata.getComment(), columnMetadata.isHidden()); }
Optional<ViewDefinition> viewDefinition = metadata.getView(session, objectName); if (!viewDefinition.isPresent()) { if (metadata.getTableHandle(session, objectName).isPresent()) { throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a view", objectName); Query query = parseView(viewDefinition.get().getOriginalSql(), objectName, node); String sql = formatSql(new CreateView(createQualifiedName(objectName), query, false), Optional.of(parameters)).trim(); return singleValueQuery("Create View", sql); Optional<TableHandle> tableHandle = metadata.getTableHandle(session, objectName); if (!tableHandle.isPresent()) { throw new SemanticException(MISSING_TABLE, node, "Table '%s' does not exist", objectName); Map<String, PropertyMetadata<?>> allColumnProperties = metadata.getColumnPropertyManager().getAllProperties().get(tableHandle.get().getConnectorId()); List<TableElement> columns = connectorTableMetadata.getColumns().stream() .filter(column -> !column.isHidden()) .map(column -> { List<Property> propertyNodes = buildProperties(objectName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties); return new ColumnDefinition(new Identifier(column.getName()), column.getType().getDisplayName(), propertyNodes, Optional.ofNullable(column.getComment())); }) .collect(toImmutableList()); Map<String, Object> properties = connectorTableMetadata.getProperties(); Map<String, PropertyMetadata<?>> allTableProperties = metadata.getTablePropertyManager().getAllProperties().get(tableHandle.get().getConnectorId()); List<Property> propertyNodes = buildProperties(objectName, Optional.empty(), INVALID_TABLE_PROPERTY, properties, allTableProperties);
.filter(column -> column.getName().equals("ds")) .map(ColumnMetadata::getName) .collect(toList()); queryId = session.getQueryId(); ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, createTableColumns, createTableProperties(storageFormat, partitionedBy)); metadata.createTable(session, tableMetadata, false); transaction.commit(); .map(column -> new ColumnMetadata( column.getName(), column.getType(), column.getComment(), columnExtraInfo(partitionedBy.contains(column.getName())), false)) .collect(toList()); assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), expectedColumns); Table table = transaction.getMetastore(tableName.getSchemaName()).getTable(tableName.getSchemaName(), tableName.getTableName()).get(); assertEquals(table.getStorage().getStorageFormat().getInputFormat(), storageFormat.getInputFormat()); MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEquals(result.getRowCount(), 0);
private static KdbTree loadKdbTree(String tableName, Session session, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager) QualifiedObjectName name = toQualifiedObjectName(tableName, session.getCatalog().get(), session.getSchema().get()); TableHandle tableHandle = metadata.getTableHandle(session, name) .orElseThrow(() -> new PrestoException(INVALID_SPATIAL_PARTITIONING, format("Table not found: %s", name))); Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle); List<ColumnHandle> visibleColumnHandles = columnHandles.values().stream() .filter(handle -> !metadata.getColumnMetadata(session, tableHandle, handle).isHidden()) .collect(toImmutableList()); checkSpatialPartitioningTable(visibleColumnHandles.size() == 1, "Expected single column for table %s, but found %s columns", name, columnHandles.size()); try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, ImmutableList.of(kdbTreeColumn))) { do { getFutureValue(pageSource.isBlocked());
@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata tableMetadata) { validatePartitionColumns(tableMetadata); validateBucketColumns(tableMetadata); Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return Optional.empty(); } if (!bucketProperty.get().getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) { throw new PrestoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled"); } List<String> bucketedBy = bucketProperty.get().getBucketedBy(); Map<String, HiveType> hiveTypeMap = tableMetadata.getColumns().stream() .collect(toMap(ColumnMetadata::getName, column -> toHiveType(typeTranslator, column.getType()))); return Optional.of(new ConnectorNewTableLayout( new HivePartitioningHandle( bucketProperty.get().getBucketCount(), bucketedBy.stream() .map(hiveTypeMap::get) .collect(toList()), OptionalInt.of(bucketProperty.get().getBucketCount())), bucketedBy)); }
if (metadata.getView(session, targetTable).isPresent()) { throw new SemanticException(NOT_SUPPORTED, insert, "Inserting into views is not supported"); Optional<TableHandle> targetTableHandle = metadata.getTableHandle(session, targetTable); if (!targetTableHandle.isPresent()) { throw new SemanticException(MISSING_TABLE, insert, "Table '%s' does not exist", targetTable); TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTableHandle.get()); List<String> tableColumns = tableMetadata.getColumns().stream() .filter(column -> !column.isHidden()) .map(ColumnMetadata::getName) .collect(toImmutableList()); .map(Identifier::getValue) .map(column -> column.toLowerCase(ENGLISH)) .collect(toImmutableList()); analysis.setInsert(new Analysis.Insert( targetTableHandle.get(), insertColumns.stream().map(columnHandles::get).collect(toImmutableList()))); .map(insertColumn -> tableMetadata.getColumn(insertColumn).getType()) .collect(toImmutableList());
private static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter(Table table, TypeManager typeManager) ImmutableList.Builder<String> columnNames = ImmutableList.builder(); table.getPartitionColumns().stream().map(Column::getName).forEach(columnNames::add); table.getDataColumns().stream().map(Column::getName).forEach(columnNames::add); List<String> allColumnNames = columnNames.build(); if (allColumnNames.size() > Sets.newHashSet(allColumnNames).size()) { throw new PrestoException(HIVE_INVALID_METADATA, ImmutableMap.Builder<String, Optional<String>> builder = ImmutableMap.builder(); for (Column field : concat(tableColumns, table.getPartitionColumns())) { if (field.getComment().isPresent() && !field.getComment().get().equals("from deserializer")) { builder.put(field.getName(), field.getComment()); builder.put(field.getName(), Optional.empty()); return handle -> new ColumnMetadata( handle.getName(), typeManager.getType(handle.getTypeSignature()),
throws Exception String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); Collection<Slice> fragments = getFutureValue(sink.finish()); metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of()); .filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()) .collect(toList()); ConnectorSession session = newSession(); PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(session.getUser()); Table oldTable = transaction.getMetastore(schemaName).getTable(schemaName, tableName).get(); HiveTypeTranslator hiveTypeTranslator = new HiveTypeTranslator(); List<Column> dataColumns = tableAfter.stream() .filter(columnMetadata -> !columnMetadata.getName().equals("ds")) .map(columnMetadata -> new Column(columnMetadata.getName(), toHiveType(hiveTypeTranslator, columnMetadata.getType()), Optional.empty())) .collect(toList()); Table.Builder newTable = Table.builder(oldTable) Collection<Slice> fragments = getFutureValue(sink.finish()); metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of());
if (!table.getName().getPrefix().isPresent()) { ImmutableList.Builder<Field> fieldBuilder = ImmutableList.builder(); fieldBuilder.add(Field.newQualified( fields = fieldBuilder.build(); analysis.addEmptyColumnReferencesForTable(accessControl, session.getIdentity(), name); Optional<ViewDefinition> optionalView = metadata.getView(session, name); .map(column -> Field.newQualified( .collect(toImmutableList()); Field field = Field.newQualified( table.getName(), Optional.of(column.getName()), column.getType(), column.isHidden(), Optional.of(name), Optional.of(column.getName()), false); fields.add(field); ColumnHandle columnHandle = columnHandles.get(column.getName()); checkArgument(columnHandle != null, "Unknown field %s", field); analysis.setColumn(field, columnHandle);
return Optional.empty(); return Optional.empty(); .map(HiveColumnHandle::getTypeSignature) .map(typeManager::getType) .collect(toImmutableList()); .map(column -> new ColumnMetadata( column.getName(), typeManager.getType(column.getTypeSignature()), column.getComment().orElse(null), column.isHidden())) .collect(toImmutableList()); public ConnectorTableMetadata getTableMetadata() return new ConnectorTableMetadata(tableName, partitionSystemTableColumns);
private RelationPlan createTableCreationPlan(Analysis analysis, Query query) { QualifiedObjectName destination = analysis.getCreateTableDestination().get(); RelationPlan plan = createRelationPlan(analysis, query); ConnectorTableMetadata tableMetadata = createTableMetadata( destination, getOutputTableColumns(plan, analysis.getColumnAliases()), analysis.getCreateTableProperties(), analysis.getParameters(), analysis.getCreateTableComment()); Optional<NewTableLayout> newTableLayout = metadata.getNewTableLayout(session, destination.getCatalogName(), tableMetadata); List<String> columnNames = tableMetadata.getColumns().stream() .filter(column -> !column.isHidden()) .map(ColumnMetadata::getName) .collect(toImmutableList()); TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadata(session, destination.getCatalogName(), tableMetadata); return createTableWriterPlan( analysis, plan, new CreateName(destination.getCatalogName(), tableMetadata, newTableLayout), columnNames, newTableLayout, statisticsMetadata); }
private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) Analysis.Insert insert = analysis.getInsert().get(); .filter(column -> !column.isHidden()) .collect(toImmutableList()); List<String> visibleTableColumnNames = visibleTableColumns.stream() .map(ColumnMetadata::getName) .collect(toImmutableList()); Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget()); Assignments.Builder assignments = Assignments.builder(); for (ColumnMetadata column : tableMetadata.getColumns()) { if (column.isHidden()) { continue; Symbol output = symbolAllocator.newSymbol(column.getName(), column.getType()); int index = insert.getColumns().indexOf(columns.get(column.getName())); if (index < 0) { Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString()); assignments.put(output, cast); Type tableType = column.getType(); Type queryType = symbolAllocator.getTypes().get(input); .map(column -> Field.newUnqualified(column.getName(), column.getType())) .collect(toImmutableList()); Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
ConnectorId connectorId = metadata.getCatalogHandle(session, tableName.getCatalogName()) .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName())); Type type; try { type = metadata.getType(parseTypeSignature(column.getType())); parameters); columns.put(name, new ColumnMetadata(name, type, column.getComment().orElse(null), null, false, columnProperties)); inheritedProperties = likeTableMetadata.getMetadata().getProperties(); .filter(column -> !column.isHidden()) .forEach(column -> { if (columns.containsKey(column.getName().toLowerCase(Locale.ENGLISH))) { throw new SemanticException(DUPLICATE_COLUMN_NAME, element, "Column name '%s' specified more than once", column.getName()); columns.put(column.getName().toLowerCase(Locale.ENGLISH), column); }); ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName.asSchemaTableName(), ImmutableList.copyOf(columns.values()), finalProperties, statement.getComment()); try { metadata.createTable(session, tableName.getCatalogName(), tableMetadata, statement.isNotExists());
private static List<AccumuloColumnHandle> getColumnHandles(ConnectorTableMetadata meta, String rowIdColumn) Map<String, Pair<String, String>> mapping = AccumuloTableProperties.getColumnMapping(meta.getProperties()).orElse(autoGenerateMapping(meta.getColumns(), AccumuloTableProperties.getLocalityGroups(meta.getProperties()))); if (cm.getName().equalsIgnoreCase(rowIdColumn)) { cBuilder.add( new AccumuloColumnHandle( rowIdColumn, Optional.empty(), Optional.empty(), cm.getType(), ordinal, "Accumulo row ID", if (!mapping.containsKey(cm.getName())) { throw new InvalidParameterException(format("Misconfigured mapping for presto column %s", cm.getName())); Pair<String, String> famqual = mapping.get(cm.getName()); boolean indexed = indexedColumns.isPresent() && indexedColumns.get().contains(cm.getName().toLowerCase(Locale.ENGLISH)); String comment = format("Accumulo column %s:%s. Indexed: %b", famqual.getLeft(), famqual.getRight(), indexed); cBuilder.add( new AccumuloColumnHandle( cm.getName(), Optional.of(famqual.getLeft()), Optional.of(famqual.getRight()), cm.getType(), ordinal, comment,