/** * Get the physical layout for a inserting into an existing table. */ default Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) { List<ConnectorTableLayout> layouts = getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), map -> true), Optional.empty()) .stream() .map(ConnectorTableLayoutResult::getTableLayout) .filter(layout -> layout.getTablePartitioning().isPresent()) .collect(toList()); if (layouts.isEmpty()) { return Optional.empty(); } if (layouts.size() > 1) { throw new PrestoException(NOT_SUPPORTED, "Tables with multiple layouts can not be written"); } ConnectorTableLayout layout = layouts.get(0); ConnectorPartitioningHandle partitioningHandle = layout.getTablePartitioning().get().getPartitioningHandle(); Map<ColumnHandle, String> columnNamesByHandle = getColumnHandles(session, tableHandle).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); List<String> partitionColumns = layout.getTablePartitioning().get().getPartitioningColumns().stream() .map(columnNamesByHandle::get) .collect(toList()); return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionColumns)); }
private static void assertInvalidExtract(String inputJson, String jsonPath, String message) { try { doJsonExtract(inputJson, jsonPath); } catch (PrestoException e) { assertEquals(e.getErrorCode(), INVALID_FUNCTION_ARGUMENT.toErrorCode()); assertEquals(e.getMessage(), message); } } }
public static Signature resolveFunction(FunctionCall node, List<TypeSignatureProvider> argumentTypes, FunctionRegistry functionRegistry) { try { return functionRegistry.resolveFunction(node.getName(), argumentTypes); } catch (PrestoException e) { if (e.getErrorCode().getCode() == StandardErrorCode.FUNCTION_NOT_FOUND.toErrorCode().getCode()) { throw new SemanticException(SemanticErrorCode.FUNCTION_NOT_FOUND, node, e.getMessage()); } if (e.getErrorCode().getCode() == StandardErrorCode.AMBIGUOUS_FUNCTION_CALL.toErrorCode().getCode()) { throw new SemanticException(SemanticErrorCode.AMBIGUOUS_FUNCTION_CALL, node, e.getMessage()); } throw e; } }
private static void validatePartitionColumns(ConnectorTableMetadata tableMetadata) { List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toList()); if (!allColumns.containsAll(partitionedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Partition columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(partitionedBy), ImmutableSet.copyOf(allColumns)))); } if (allColumns.size() == partitionedBy.size()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Table contains only partition columns"); } if (!allColumns.subList(allColumns.size() - partitionedBy.size(), allColumns.size()).equals(partitionedBy)) { throw new PrestoException(HIVE_COLUMN_ORDER_MISMATCH, "Partition keys must be the last columns in the table and in the same order as the table properties: " + partitionedBy); } }
private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) { List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries) .map(this::createQueryMemoryInfo) .collect(toImmutableList()); List<MemoryInfo> nodeMemoryInfos = nodes.values().stream() .map(RemoteNodeMemory::getInfo) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos); if (chosenQueryId.isPresent()) { log.debug("Low memory killer chose %s", chosenQueryId.get()); Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional()); if (chosenQuery.isPresent()) { // See comments in isLastKilledQueryGone for why chosenQuery might be absent. chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = chosenQueryId.get(); logQueryKill(chosenQueryId.get(), nodeMemoryInfos); } } }
@Test public void testCreateTableInNotExistSchema() { SchemaTableName schemaTableName = new SchemaTableName("schema1", "test_table"); try { metadata.beginCreateTable(SESSION, new ConnectorTableMetadata(schemaTableName, ImmutableList.of(), tableProperties), Optional.empty()); fail("Should fail because schema does not exist"); } catch (PrestoException ex) { assertEquals(ex.getErrorCode(), NOT_FOUND.toErrorCode()); assertTrue(ex.getMessage().equals("Schema schema1 not found")); } }
public static Optional<HiveBucketProperty> fromStorageDescriptor(StorageDescriptor storageDescriptor, String tablePartitionName) { boolean bucketColsSet = storageDescriptor.isSetBucketCols() && !storageDescriptor.getBucketCols().isEmpty(); boolean numBucketsSet = storageDescriptor.isSetNumBuckets() && storageDescriptor.getNumBuckets() > 0; if (!numBucketsSet) { // In Hive, a table is considered as not bucketed when its bucketCols is set but its numBucket is not set. return Optional.empty(); } if (!bucketColsSet) { throw new PrestoException(HIVE_INVALID_METADATA, "Table/partition metadata has 'numBuckets' set, but 'bucketCols' is not set: " + tablePartitionName); } List<SortingColumn> sortedBy = ImmutableList.of(); if (storageDescriptor.isSetSortCols()) { sortedBy = storageDescriptor.getSortCols().stream() .map(order -> SortingColumn.fromMetastoreApiOrder(order, tablePartitionName)) .collect(toImmutableList()); } return Optional.of(new HiveBucketProperty(storageDescriptor.getBucketCols(), storageDescriptor.getNumBuckets(), sortedBy)); }
public List<File> files() { checkState(location.exists(), "location %s doesn't exist", location); if (!pattern.isPresent()) { return ImmutableList.of(location); } checkState(location.isDirectory(), "location %s is not a directory", location); try (DirectoryStream<Path> paths = newDirectoryStream(location.toPath(), pattern.get())) { ImmutableList.Builder<File> builder = ImmutableList.builder(); for (Path path : paths) { builder.add(path.toFile()); } List<File> files = builder.build(); if (files.isEmpty()) { throw new PrestoException(LOCAL_FILE_NO_FILES, "No matching files found in directory: " + location); } return files.stream() .sorted((o1, o2) -> Long.compare(o2.lastModified(), o1.lastModified())) .collect(Collectors.toList()); } catch (IOException e) { throw new PrestoException(LOCAL_FILE_FILESYSTEM_ERROR, "Error listing files in directory: " + location, e); } } }
private static void validateBucketColumns(ConnectorTableMetadata tableMetadata) { Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return; } Set<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet()); List<String> bucketedBy = bucketProperty.get().getBucketedBy(); if (!allColumns.containsAll(bucketedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Bucketing columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(bucketedBy), ImmutableSet.copyOf(allColumns)))); } List<String> sortedBy = bucketProperty.get().getSortedBy().stream() .map(SortingColumn::getColumnName) .collect(toImmutableList()); if (!allColumns.containsAll(sortedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Sorting columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(sortedBy), ImmutableSet.copyOf(allColumns)))); } }
private static AbstractTableMetadata getTableMetadata(KeyspaceMetadata keyspace, String caseInsensitiveTableName) { List<AbstractTableMetadata> tables = Stream.concat( keyspace.getTables().stream(), keyspace.getMaterializedViews().stream()) .filter(table -> table.getName().equalsIgnoreCase(caseInsensitiveTableName)) .collect(toImmutableList()); if (tables.size() == 0) { throw new TableNotFoundException(new SchemaTableName(keyspace.getName(), caseInsensitiveTableName)); } else if (tables.size() == 1) { return tables.get(0); } String tableNames = tables.stream() .map(AbstractTableMetadata::getName) .sorted() .collect(joining(", ")); throw new PrestoException( NOT_SUPPORTED, format("More than one table has been found for the case insensitive table name: %s -> (%s)", caseInsensitiveTableName, tableNames)); }
public static Optional<HiveBucketHandle> getHiveBucketHandle(Table table) { Optional<HiveBucketProperty> hiveBucketProperty = table.getStorage().getBucketProperty(); if (!hiveBucketProperty.isPresent()) { return Optional.empty(); } Map<String, HiveColumnHandle> map = getRegularColumnHandles(table).stream() .collect(Collectors.toMap(HiveColumnHandle::getName, identity())); ImmutableList.Builder<HiveColumnHandle> bucketColumns = ImmutableList.builder(); for (String bucketColumnName : hiveBucketProperty.get().getBucketedBy()) { HiveColumnHandle bucketColumnHandle = map.get(bucketColumnName); if (bucketColumnHandle == null) { throw new PrestoException( HIVE_INVALID_METADATA, format("Table '%s.%s' is bucketed on non-existent column '%s'", table.getDatabaseName(), table.getTableName(), bucketColumnName)); } bucketColumns.add(bucketColumnHandle); } int bucketCount = hiveBucketProperty.get().getBucketCount(); return Optional.of(new HiveBucketHandle(bucketColumns.build(), bucketCount, bucketCount)); }
@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession connectorSession, ConnectorTableMetadata tableMetadata) { List<String> distributeColumns = (List<String>) tableMetadata.getProperties().get(DISTRIBUTED_ON); if (distributeColumns.isEmpty()) { return Optional.empty(); } Set<String> undefinedColumns = Sets.difference( ImmutableSet.copyOf(distributeColumns), tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet())); if (!undefinedColumns.isEmpty()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Distribute columns not defined on table: " + undefinedColumns); } return Optional.of(new ConnectorNewTableLayout(BlackHolePartitioningHandle.INSTANCE, distributeColumns)); }
private Map<HivePartitionName, PartitionStatistics> loadPartitionColumnStatistics(Iterable<? extends HivePartitionName> keys) { SetMultimap<HiveTableName, HivePartitionName> tablePartitions = stream(keys) .collect(toImmutableSetMultimap(HivePartitionName::getHiveTableName, key -> key)); ImmutableMap.Builder<HivePartitionName, PartitionStatistics> result = ImmutableMap.builder(); tablePartitions.keySet().forEach(table -> { Set<String> partitionNames = tablePartitions.get(table).stream() .map(partitionName -> partitionName.getPartitionName().get()) .collect(toImmutableSet()); Map<String, PartitionStatistics> partitionStatistics = delegate.getPartitionStatistics(table.getDatabaseName(), table.getTableName(), partitionNames); for (String partitionName : partitionNames) { if (!partitionStatistics.containsKey(partitionName)) { throw new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + partitionName); } result.put(HivePartitionName.hivePartitionName(table, partitionName), partitionStatistics.get(partitionName)); } }); return result.build(); }
@Override public Optional<SelectionContext<VariableMap>> match(SelectionCriteria criteria) { if (lastRefresh.get() == 0) { throw new PrestoException(CONFIGURATION_UNAVAILABLE, "Selectors cannot be fetched from database"); } if (selectors.get().isEmpty()) { throw new PrestoException(CONFIGURATION_INVALID, "No selectors are configured"); } return selectors.get().stream() .map(s -> s.match(criteria)) .filter(Optional::isPresent) .map(Optional::get) .findFirst(); }
@Override public CompletableFuture<Collection<Slice>> finish() { try { // Done serializing rows, so flush and close the writer and indexer writer.flush(); writer.close(); if (indexer.isPresent()) { indexer.get().close(); } } catch (MutationsRejectedException e) { throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server on flush", e); } // TODO Look into any use of the metadata for writing out the rows return completedFuture(ImmutableList.of()); }
public Signature resolveOperator(OperatorType operatorType, List<? extends Type> argumentTypes) throws OperatorNotFoundException { try { return resolveFunction(QualifiedName.of(mangleOperatorName(operatorType)), fromTypes(argumentTypes)); } catch (PrestoException e) { if (e.getErrorCode().getCode() == FUNCTION_NOT_FOUND.toErrorCode().getCode()) { throw new OperatorNotFoundException( operatorType, argumentTypes.stream() .map(Type::getTypeSignature) .collect(toImmutableList())); } else { throw e; } } }
QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName()); Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName); if (!tableHandle.isPresent()) { throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName); .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName())); accessControl.checkCanAddColumns(session.getRequiredTransactionId(), session.getIdentity(), tableName); Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle.get());
@Test public void testFailedTransactionWorkflow() assertEquals(transactionManager.getAllTransactionInfos().size(), 1); TransactionInfo transactionInfo = transactionManager.getTransactionInfo(transactionId); assertFalse(transactionInfo.isAutoCommitContext()); assertTrue(transactionInfo.getConnectorIds().isEmpty()); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); ConnectorMetadata metadata = transactionManager.getOptionalCatalogMetadata(transactionId, CATALOG_NAME).get().getMetadata(); metadata.listSchemaNames(TEST_SESSION.toConnectorSession(CONNECTOR_ID)); transactionInfo = transactionManager.getTransactionInfo(transactionId); assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of(CONNECTOR_ID, INFORMATION_SCHEMA_ID, SYSTEM_TABLES_ID)); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); assertEquals(e.getErrorCode(), TRANSACTION_ALREADY_ABORTED.toErrorCode()); assertEquals(transactionManager.getAllTransactionInfos().size(), 1);
private void checkTableNotExists(SchemaTableName tableName) { if (tables.values().stream() .map(MemoryTableHandle::toSchemaTableName) .anyMatch(tableName::equals)) { throw new PrestoException(ALREADY_EXISTS, format("Table [%s] already exists", tableName.toString())); } if (views.keySet().contains(tableName)) { throw new PrestoException(ALREADY_EXISTS, format("View [%s] already exists", tableName.toString())); } }
@Override public ListenableFuture<?> execute(Rollback statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List<Expression> parameters) { Session session = stateMachine.getSession(); if (!session.getTransactionId().isPresent()) { throw new PrestoException(NOT_IN_TRANSACTION, "No transaction in progress"); } TransactionId transactionId = session.getTransactionId().get(); stateMachine.clearTransactionId(); transactionManager.asyncAbort(transactionId); return immediateFuture(null); }