@Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { TpcdsTableLayoutHandle layout = (TpcdsTableLayoutHandle) handle; return getTableLayouts(session, layout.getTable(), Constraint.alwaysTrue(), Optional.empty()) .get(0) .getTableLayout(); }
@Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { TpchTableLayoutHandle layout = (TpchTableLayoutHandle) handle; // tables in this connector have a single layout return getTableLayouts(session, layout.getTable(), Constraint.alwaysTrue(), Optional.empty()) .get(0) .getTableLayout(); }
@Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { MongoTableLayoutHandle layout = (MongoTableLayoutHandle) handle; // tables in this connector have a single layout return getTableLayouts(session, layout.getTable(), Constraint.alwaysTrue(), Optional.empty()) .get(0) .getTableLayout(); }
@Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { RedisTableLayoutHandle layout = convertLayout(handle); // tables in this connector have a single layout return getTableLayouts(session, layout.getTable(), Constraint.alwaysTrue(), Optional.empty()) .get(0) .getTableLayout(); }
private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
@Override public List<TableLayoutResult> getLayouts(Session session, TableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { if (constraint.getSummary().isNone()) { return ImmutableList.of(); } ConnectorId connectorId = table.getConnectorId(); ConnectorTableHandle connectorTable = table.getConnectorHandle(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); ConnectorSession connectorSession = session.toConnectorSession(connectorId); List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(connectorSession, connectorTable, constraint, desiredColumns); return layouts.stream() .map(layout -> new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layout.getTableLayout()), layout.getUnenforcedConstraint())) .collect(toImmutableList()); }
@Test public void testGetPartitionNamesUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertEquals(getAllPartitions(getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).size(), 1); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), unpartitionedTableLayout); } }
@Test public void testReadTableBeforeCreationCompleted() { assertNoTables(); SchemaTableName tableName = new SchemaTableName("default", "temp_table"); ConnectorOutputTableHandle table = metadata.beginCreateTable( SESSION, new ConnectorTableMetadata(tableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty()); List<SchemaTableName> tableNames = metadata.listTables(SESSION, Optional.empty()); assertTrue(tableNames.size() == 1, "Expected exactly one table"); ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, tableName); List<ConnectorTableLayoutResult> tableLayouts = metadata.getTableLayouts(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertTrue(tableLayouts.size() == 1, "Expected exactly one layout."); ConnectorTableLayout tableLayout = tableLayouts.get(0).getTableLayout(); ConnectorTableLayoutHandle tableLayoutHandle = tableLayout.getHandle(); assertTrue(tableLayoutHandle instanceof MemoryTableLayoutHandle); assertTrue(((MemoryTableLayoutHandle) tableLayoutHandle).getDataFragments().isEmpty(), "Data fragments should be empty"); metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of()); }
@Test public void testGetPartitionNames() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
@Test public void testGetPartitions() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
@Test public void testGetPartitionsWithBindings() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L)))), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
@Test public void testGetPartitionSplitsBatch() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING); assertEquals(getSplitCount(splitSource), partitionCount); } }
@Test public void testGetPartitionSplitsBatchUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING); assertEquals(getSplitCount(splitSource), 1); } }
@Test public void testGetPartitionSplitsTableNotReadablePartition() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableNotReadable); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING)); fail("Expected HiveNotReadableException"); } catch (HiveNotReadableException e) { assertThat(e).hasMessageMatching("Table '.*\\.presto_test_not_readable' is not readable: reason for not readable"); assertEquals(e.getTableName(), tableNotReadable); assertEquals(e.getPartition(), Optional.empty()); } } }
/** * Tests information schema predicate pushdown when both schema and table name are specified. */ @Test public void testInformationSchemaPredicatePushdown() { TransactionId transactionId = transactionManager.beginTransaction(false); ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>(); domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema"))); domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view"))); Constraint<ColumnHandle> constraint = new Constraint<>(TupleDomain.withColumnDomains(domains.build())); InformationSchemaMetadata informationSchemaMetadata = new InformationSchemaMetadata("test_catalog", metadata); List<ConnectorTableLayoutResult> layoutResults = informationSchemaMetadata.getTableLayouts( createNewSession(transactionId), new InformationSchemaTableHandle("test_catalog", "information_schema", "views"), constraint, Optional.empty()); assertEquals(layoutResults.size(), 1); ConnectorTableLayoutHandle handle = layoutResults.get(0).getTableLayout().getHandle(); assertTrue(handle instanceof InformationSchemaTableLayoutHandle); InformationSchemaTableLayoutHandle tableHandle = (InformationSchemaTableLayoutHandle) handle; assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view"))); }
private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); } }
@Test(enabled = false) public void testPartitionSchemaNonCanonical() throws Exception { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle table = getTableHandle(metadata, tablePartitionSchemaChangeNonCanonical); ColumnHandle column = metadata.getColumnHandles(session, table).get("t_boolean"); assertNotNull(column); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, table, new Constraint<>(TupleDomain.fromFixedValues(ImmutableMap.of(column, NullableValue.of(BOOLEAN, false)))), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(getAllPartitions(layoutHandle).size(), 1); assertEquals(getPartitionId(getAllPartitions(layoutHandle).get(0)), "t_boolean=0"); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); ImmutableList<ColumnHandle> columnHandles = ImmutableList.of(column); try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { fail("expected exception"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), HIVE_INVALID_PARTITION_VALUE.toErrorCode()); } } }
@Test public void testGetPartitionSplitsTableOfflinePartition() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableOfflinePartition); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30")); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING)); fail("Expected PartitionOfflineException"); } catch (PartitionOfflineException e) { assertEquals(e.getTableName(), tableOfflinePartition); assertEquals(e.getPartition(), "ds=2012-12-30"); } } }
@Test public void testGetRecords() throws Exception { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle table = getTableHandle(metadata, this.table); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, table).values()); Map<String, Integer> columnIndex = indexColumns(columnHandles); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, table, Constraint.alwaysTrue(), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING); long sum = 0; for (ConnectorSplit split : getAllSplits(splitSource)) { try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); for (MaterializedRow row : result) { sum += (Long) row.getField(columnIndex.get("t_bigint")); } } } // The test table is made up of multiple S3 objects with same data and different compression codec // formats: uncompressed | .gz | .lz4 | .bz2 assertEquals(sum, 78300 * 4); } }
@Test public void testOrdersOrderStatusPredicatePushdown() { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", ORDERS.getTableName())); TupleDomain<ColumnHandle> domain; ConnectorTableLayoutResult tableLayout; domain = fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P")); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); domain = fixedValueTupleDomain(tpchMetadata, ORDER_KEY, 42L); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), domain, session); assertTupleDomainEquals( tableLayout.getTableLayout().getPredicate(), // The most important thing about the expected value that it is NOT TupleDomain.none() (or equivalent). // Using concrete expected value instead of checking TupleDomain::isNone to make sure the test doesn't pass on some other wrong value. TupleDomain.columnWiseUnion( fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("F")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("O")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P"))), session); }