@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { MongoTableHandle tableHandle = (MongoTableHandle) table; Optional<Set<ColumnHandle>> partitioningColumns = Optional.empty(); //TODO: sharding key ImmutableList.Builder<LocalProperty<ColumnHandle>> localProperties = ImmutableList.builder(); MongoTable tableInfo = mongoSession.getTable(tableHandle.getSchemaTableName()); Map<String, ColumnHandle> columns = getColumnHandles(session, tableHandle); for (MongoIndex index : tableInfo.getIndexes()) { for (MongodbIndexKey key : index.getKeys()) { if (!key.getSortOrder().isPresent()) { continue; } if (columns.get(key.getName()) != null) { localProperties.add(new SortingProperty<>(columns.get(key.getName()), key.getSortOrder().get())); } } } ConnectorTableLayout layout = new ConnectorTableLayout( new MongoTableLayoutHandle(tableHandle, constraint.getSummary()), Optional.empty(), TupleDomain.all(), Optional.empty(), partitioningColumns, Optional.empty(), localProperties.build()); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); }
@Test public void testGetPartitionNamesUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertEquals(getAllPartitions(getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).size(), 1); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), unpartitionedTableLayout); } }
@Override public List<TableLayoutResult> getLayouts(Session session, TableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { if (constraint.getSummary().isNone()) { return ImmutableList.of(); } ConnectorId connectorId = table.getConnectorId(); ConnectorTableHandle connectorTable = table.getConnectorHandle(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); ConnectorSession connectorSession = session.toConnectorSession(connectorId); List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(connectorSession, connectorTable, constraint, desiredColumns); return layouts.stream() .map(layout -> new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layout.getTableLayout()), layout.getUnenforcedConstraint())) .collect(toImmutableList()); }
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
.orElseThrow(() -> new AssertionError("Table does not exist: " + tableName)); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedResultBuilder.build().getMaterializedRows()); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2015-07-03")))); Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle); TupleDomain<ColumnHandle> tupleDomain2 = TupleDomain.withColumnDomains( ImmutableMap.of(dsColumnHandle, Domain.create(ValueSet.ofRanges(Range.range(createUnboundedVarcharType(), utf8Slice("2015-07-01"), true, utf8Slice("2015-07-02"), true)), false))); Constraint<ColumnHandle> constraint2 = new Constraint<>(tupleDomain2, convertToPredicate(tupleDomain2)); List<ConnectorTableLayoutResult> tableLayoutResults2 = metadata.getTableLayouts(session, tableHandle, constraint2, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle2 = getOnlyElement(tableLayoutResults2).getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle2); ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); MaterializedResult actualAfterDelete2 = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(actualAfterDelete2.getMaterializedRows(), ImmutableList.of());
@Test public void testGetPartitionsWithBindings() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L)))), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
@Override public List<ConnectorTableLayoutResult> getTableLayouts( ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { TpcdsTableHandle tableHandle = (TpcdsTableHandle) table; ConnectorTableLayout layout = new ConnectorTableLayout( new TpcdsTableLayoutHandle(tableHandle), Optional.empty(), TupleDomain.all(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of()); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); }
@Test public void testGetRecordsS3() throws Exception { HiveTransactionHandle transaction = new HiveTransactionHandle(); HiveMetadata metadata = metadataFactory.create(); ConnectorTableHandle table = getTableHandle(metadata, tableS3); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(SESSION, table).values()); Map<String, Integer> columnIndex = indexColumns(columnHandles); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(SESSION, table, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction, SESSION, layoutHandle); long sum = 0; for (ConnectorSplit split : getAllSplits(splitSource)) { try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction, SESSION, split, columnHandles)) { MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, getTypes(columnHandles)); for (MaterializedRow row : result) { sum += (Long) row.getField(columnIndex.get("t_bigint")); } } } assertEquals(sum, 78300); }
private void createTable(SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception List<ColumnMetadata> columns = ImmutableList.<ColumnMetadata>builder() .add(new ColumnMetadata("id", BIGINT)) .build(); ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty()); metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of()); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { AtopTableHandle tableHandle = (AtopTableHandle) table; Optional<Map<ColumnHandle, Domain>> domains = constraint.getSummary().getDomains(); Domain endTimeDomain = Domain.all(TIMESTAMP_WITH_TIME_ZONE); Domain startTimeDomain = Domain.all(TIMESTAMP_WITH_TIME_ZONE); if (domains.isPresent()) { if (domains.get().containsKey(START_TIME_HANDLE)) { startTimeDomain = domains.get().get(START_TIME_HANDLE); } if (domains.get().containsKey(END_TIME_HANDLE)) { endTimeDomain = domains.get().get(END_TIME_HANDLE); } } AtopTableLayoutHandle layoutHandle = new AtopTableLayoutHandle(tableHandle, startTimeDomain, endTimeDomain); ConnectorTableLayout tableLayout = getTableLayout(session, layoutHandle); return ImmutableList.of(new ConnectorTableLayoutResult(tableLayout, constraint.getSummary())); }
Optional<Set<ColumnHandle>> desiredColumns ) { ImmutableList.Builder<EthereumBlockRange> builder = ImmutableList.builder(); Optional<Map<ColumnHandle, Domain>> domains = constraint.getSummary().getDomains(); if (domains.isPresent()) { Map<ColumnHandle, Domain> columnHandleDomainMap = domains.get(); for (Map.Entry<ColumnHandle, Domain> entry : columnHandleDomainMap.entrySet()) { if (entry.getKey() instanceof EthereumColumnHandle ConnectorTableLayout layout = new ConnectorTableLayout(new EthereumTableLayoutHandle(handle, builder.build())); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary()));
@Test(enabled = false) public void testPartitionSchemaNonCanonical() throws Exception { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle table = getTableHandle(metadata, tablePartitionSchemaChangeNonCanonical); ColumnHandle column = metadata.getColumnHandles(session, table).get("t_boolean"); assertNotNull(column); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, table, new Constraint<>(TupleDomain.fromFixedValues(ImmutableMap.of(column, NullableValue.of(BOOLEAN, false)))), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(getAllPartitions(layoutHandle).size(), 1); assertEquals(getPartitionId(getAllPartitions(layoutHandle).get(0)), "t_boolean=0"); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); ImmutableList<ColumnHandle> columnHandles = ImmutableList.of(column); try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { fail("expected exception"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), HIVE_INVALID_PARTITION_VALUE.toErrorCode()); } } }
private MaterializedResult readTable( Transaction transaction, ConnectorTableHandle tableHandle, List<ColumnHandle> columnHandles, ConnectorSession session, TupleDomain<ColumnHandle> tupleDomain, OptionalInt expectedSplitCount, Optional<HiveStorageFormat> expectedStorageFormat) throws Exception { List<ConnectorTableLayoutResult> tableLayoutResults = transaction.getMetadata().getTableLayouts( session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); List<ConnectorSplit> splits = getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); if (expectedSplitCount.isPresent()) { assertEquals(splits.size(), expectedSplitCount.getAsInt()); } ImmutableList.Builder<MaterializedRow> allRows = ImmutableList.builder(); for (ConnectorSplit split : splits) { try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { expectedStorageFormat.ifPresent(format -> assertPageSourceType(pageSource, format)); MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); allRows.addAll(result.getMaterializedRows()); } } return new MaterializedResult(allRows.build(), getTypes(columnHandles)); }
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { JdbcTableHandle tableHandle = (JdbcTableHandle) table; ConnectorTableLayout layout = new ConnectorTableLayout(new JdbcTableLayoutHandle(tableHandle, constraint.getSummary())); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); }
@Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { TpcdsTableLayoutHandle layout = (TpcdsTableLayoutHandle) handle; return getTableLayouts(session, layout.getTable(), Constraint.alwaysTrue(), Optional.empty()) .get(0) .getTableLayout(); }
@Override public synchronized List<ConnectorTableLayoutResult> getTableLayouts( ConnectorSession session, ConnectorTableHandle handle, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { requireNonNull(handle, "handle is null"); checkArgument(handle instanceof MemoryTableHandle); MemoryTableHandle memoryTableHandle = (MemoryTableHandle) handle; checkState( tableDataFragments.containsKey(memoryTableHandle.getTableId()), "Inconsistent state for the table [%s.%s]", memoryTableHandle.getSchemaName(), memoryTableHandle.getTableName()); List<MemoryDataFragment> expectedFragments = ImmutableList.copyOf( tableDataFragments.get(memoryTableHandle.getTableId()).values()); MemoryTableLayoutHandle layoutHandle = new MemoryTableLayoutHandle(memoryTableHandle, expectedFragments); return ImmutableList.of(new ConnectorTableLayoutResult(getTableLayout(session, layoutHandle), constraint.getSummary())); }
@Test public void testReadTableBeforeCreationCompleted() { assertNoTables(); SchemaTableName tableName = new SchemaTableName("default", "temp_table"); ConnectorOutputTableHandle table = metadata.beginCreateTable( SESSION, new ConnectorTableMetadata(tableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty()); List<SchemaTableName> tableNames = metadata.listTables(SESSION, Optional.empty()); assertTrue(tableNames.size() == 1, "Expected exactly one table"); ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, tableName); List<ConnectorTableLayoutResult> tableLayouts = metadata.getTableLayouts(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertTrue(tableLayouts.size() == 1, "Expected exactly one layout."); ConnectorTableLayout tableLayout = tableLayouts.get(0).getTableLayout(); ConnectorTableLayoutHandle tableLayoutHandle = tableLayout.getHandle(); assertTrue(tableLayoutHandle instanceof MemoryTableLayoutHandle); assertTrue(((MemoryTableLayoutHandle) tableLayoutHandle).getDataFragments().isEmpty(), "Data fragments should be empty"); metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of()); }
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { HiveTableHandle handle = (HiveTableHandle) tableHandle; HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint); return ImmutableList.of(new ConnectorTableLayoutResult( getTableLayout( session, new HiveTableLayoutHandle( handle.getSchemaTableName(), ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), getPartitionsAsList(hivePartitionResult), hivePartitionResult.getCompactEffectivePredicate(), hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter())), hivePartitionResult.getUnenforcedConstraint())); }
@Override public List<ConnectorTableLayoutResult> getTableLayouts( ConnectorSession session, ConnectorTableHandle handle, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { BlackHoleTableHandle blackHoleHandle = (BlackHoleTableHandle) handle; BlackHoleTableLayoutHandle layoutHandle = new BlackHoleTableLayoutHandle( blackHoleHandle.getSplitCount(), blackHoleHandle.getPagesPerSplit(), blackHoleHandle.getRowsPerPage(), blackHoleHandle.getFieldsLength(), blackHoleHandle.getPageProcessingDelay()); return ImmutableList.of(new ConnectorTableLayoutResult(getTableLayout(session, layoutHandle), constraint.getSummary())); }