public static <V> Constraint<V> alwaysFalse() { return new Constraint<>(TupleDomain.<V>none(), Optional.of(bindings -> false)); }
public static <V> Constraint<V> alwaysTrue() { return new Constraint<>(TupleDomain.<V>all(), Optional.empty()); }
private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
@Override public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) { TupleDomain<ColumnHandle> targetTupleDomain = constraint.transform(fieldIdToColumnHandle::get); Predicate<Map<ColumnHandle, NullableValue>> targetPredicate = convertToPredicate(targetTupleDomain); Constraint<ColumnHandle> targetConstraint = new Constraint<>(targetTupleDomain, targetPredicate); Iterable<List<Object>> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint).getPartitions()) .map(hivePartition -> (List<Object>) IntStream.range(0, partitionColumns.size()) .mapToObj(fieldIdToColumnHandle::get) .map(columnHandle -> hivePartition.getKeys().get(columnHandle).getValue()) .collect(toList())) .iterator(); return new InMemoryRecordSet(partitionColumnTypes, records).cursor(); } };
private Constraint<ColumnHandle> getConstraint(Plan plan) { Optional<TableScanNode> scanNode = searchFrom(plan.getRoot()) .where(TableScanNode.class::isInstance) .findSingle(); if (!scanNode.isPresent()) { return Constraint.alwaysFalse(); } return new Constraint<>(scanNode.get().getCurrentConstraint()); }
private Constraint<ColumnHandle> getConstraint(Plan plan) { Optional<TableScanNode> scanNode = searchFrom(plan.getRoot()) .where(TableScanNode.class::isInstance) .findSingle(); if (!scanNode.isPresent()) { return Constraint.alwaysFalse(); } return new Constraint<>(scanNode.get().getCurrentConstraint()); }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator Constraint<ColumnHandle> constraint = new Constraint<>(node.getCurrentConstraint()); TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint); Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>(); for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) { Symbol symbol = entry.getKey(); Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue())); outputSymbolStats.put(symbol, columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics)).orElse(SymbolStatsEstimate.unknown())); } return Optional.of(PlanNodeStatsEstimate.builder() .setOutputRowCount(tableStatistics.getRowCount().getValue()) .addSymbolStatistics(outputSymbolStats) .build()); }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator Constraint<ColumnHandle> constraint = new Constraint<>(node.getCurrentConstraint()); TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint); Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>(); for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) { Symbol symbol = entry.getKey(); Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue())); outputSymbolStats.put(symbol, columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics)).orElse(SymbolStatsEstimate.unknown())); } return Optional.of(PlanNodeStatsEstimate.builder() .setOutputRowCount(tableStatistics.getRowCount().getValue()) .addSymbolStatistics(outputSymbolStats) .build()); }
private Constraint<ColumnHandle> constraint(TpchColumn<?> column, String... values) { List<TupleDomain<ColumnHandle>> valueDomains = stream(values) .map(value -> fixedValueTupleDomain(tpchMetadata, column, utf8Slice(value))) .collect(toList()); TupleDomain<ColumnHandle> domain = TupleDomain.columnWiseUnion(valueDomains); return new Constraint<>(domain, convertToPredicate(domain, column)); }
private Constraint<ColumnHandle> constraint(TpchColumn<?> column, String... values) { List<TupleDomain<ColumnHandle>> valueDomains = stream(values) .map(value -> fixedValueTupleDomain(tpchMetadata, column, utf8Slice(value))) .collect(toList()); TupleDomain<ColumnHandle> domain = TupleDomain.columnWiseUnion(valueDomains); return new Constraint<>(domain, convertToPredicate(domain, column)); }
@Test public void testGetPartitionsWithBindings() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L)))), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
@Test public void testGetPartitionsWithBindings() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L)))), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
/** * Tests information schema predicate pushdown when both schema and table name are specified. */ @Test public void testInformationSchemaPredicatePushdown() { TransactionId transactionId = transactionManager.beginTransaction(false); ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>(); domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema"))); domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view"))); Constraint<ColumnHandle> constraint = new Constraint<>(TupleDomain.withColumnDomains(domains.build())); InformationSchemaMetadata informationSchemaMetadata = new InformationSchemaMetadata("test_catalog", metadata); List<ConnectorTableLayoutResult> layoutResults = informationSchemaMetadata.getTableLayouts( createNewSession(transactionId), new InformationSchemaTableHandle("test_catalog", "information_schema", "views"), constraint, Optional.empty()); assertEquals(layoutResults.size(), 1); ConnectorTableLayoutHandle handle = layoutResults.get(0).getTableLayout().getHandle(); assertTrue(handle instanceof InformationSchemaTableLayoutHandle); InformationSchemaTableLayoutHandle tableHandle = (InformationSchemaTableLayoutHandle) handle; assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view"))); }
/** * Tests information schema predicate pushdown when both schema and table name are specified. */ @Test public void testInformationSchemaPredicatePushdown() { TransactionId transactionId = transactionManager.beginTransaction(false); ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>(); domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema"))); domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view"))); Constraint<ColumnHandle> constraint = new Constraint<>(TupleDomain.withColumnDomains(domains.build())); InformationSchemaMetadata informationSchemaMetadata = new InformationSchemaMetadata("test_catalog", metadata); List<ConnectorTableLayoutResult> layoutResults = informationSchemaMetadata.getTableLayouts( createNewSession(transactionId), new InformationSchemaTableHandle("test_catalog", "information_schema", "views"), constraint, Optional.empty()); assertEquals(layoutResults.size(), 1); ConnectorTableLayoutHandle handle = layoutResults.get(0).getTableLayout().getHandle(); assertTrue(handle instanceof InformationSchemaTableLayoutHandle); InformationSchemaTableLayoutHandle tableHandle = (InformationSchemaTableLayoutHandle) handle; assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view"))); }
private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); } }
private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); } }
@Test public void testGetPartitionSplitsTableOfflinePartition() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableOfflinePartition); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30")); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING)); fail("Expected PartitionOfflineException"); } catch (PartitionOfflineException e) { assertEquals(e.getTableName(), tableOfflinePartition); assertEquals(e.getPartition(), "ds=2012-12-30"); } } }
@Test public void testGetPartitionSplitsTableOfflinePartition() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableOfflinePartition); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30")); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING)); fail("Expected PartitionOfflineException"); } catch (PartitionOfflineException e) { assertEquals(e.getTableName(), tableOfflinePartition); assertEquals(e.getPartition(), "ds=2012-12-30"); } } }
@Test public void testOrdersOrderStatusPredicatePushdown() { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", ORDERS.getTableName())); TupleDomain<ColumnHandle> domain; ConnectorTableLayoutResult tableLayout; domain = fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P")); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); domain = fixedValueTupleDomain(tpchMetadata, ORDER_KEY, 42L); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), domain, session); assertTupleDomainEquals( tableLayout.getTableLayout().getPredicate(), // The most important thing about the expected value that it is NOT TupleDomain.none() (or equivalent). // Using concrete expected value instead of checking TupleDomain::isNone to make sure the test doesn't pass on some other wrong value. TupleDomain.columnWiseUnion( fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("F")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("O")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P"))), session); }
@Test public void testOrdersOrderStatusPredicatePushdown() { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", ORDERS.getTableName())); TupleDomain<ColumnHandle> domain; ConnectorTableLayoutResult tableLayout; domain = fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P")); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); domain = fixedValueTupleDomain(tpchMetadata, ORDER_KEY, 42L); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), domain, session); assertTupleDomainEquals( tableLayout.getTableLayout().getPredicate(), // The most important thing about the expected value that it is NOT TupleDomain.none() (or equivalent). // Using concrete expected value instead of checking TupleDomain::isNone to make sure the test doesn't pass on some other wrong value. TupleDomain.columnWiseUnion( fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("F")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("O")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P"))), session); }