Refine search
public TableScanNode( PlanNodeId id, TableHandle table, List<Symbol> outputs, Map<Symbol, ColumnHandle> assignments, Optional<TableLayoutHandle> tableLayout, TupleDomain<ColumnHandle> currentConstraint, TupleDomain<ColumnHandle> enforcedConstraint) { super(id); this.table = requireNonNull(table, "table is null"); this.outputSymbols = ImmutableList.copyOf(requireNonNull(outputs, "outputs is null")); this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs"); this.tableLayout = requireNonNull(tableLayout, "tableLayout is null"); this.currentConstraint = requireNonNull(currentConstraint, "currentConstraint is null"); this.enforcedConstraint = requireNonNull(enforcedConstraint, "enforcedConstraint is null"); if (!currentConstraint.isAll() || !enforcedConstraint.isAll()) { checkArgument(tableLayout.isPresent(), "tableLayout must be present when currentConstraint or enforcedConstraint is non-trivial"); } }
Map<ColumnHandle, NullableValue> fixedValues = TupleDomain.extractFixedValues(tupleDomain).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !indexableColumns.contains(entry.getKey())) .addAll(handleToNames(ImmutableList.copyOf(indexableColumns))) .addAll(handleToNames(ImmutableList.copyOf(fixedValues.keySet()))) .build(); if (!indexedData.getIndexedTable(tpchTableHandle.getTableName(), tpchTableHandle.getScaleFactor(), lookupColumnNames).isPresent()) { return Optional.empty(); if (!tupleDomain.isNone()) { filteredTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(tupleDomain.getDomains().get(), not(in(fixedValues.keySet())))); tpchTableHandle.getScaleFactor(), lookupColumnNames, TupleDomain.fromFixedValues(fixedValues)); return Optional.of(new ConnectorResolvedIndex(indexHandle, filteredTupleDomain));
@JsonCreator // Available for Jackson deserialization only! public static <T> TupleDomain<T> fromColumnDomains(@JsonProperty("columnDomains") Optional<List<ColumnDomain<T>>> columnDomains) { if (!columnDomains.isPresent()) { return none(); } return withColumnDomains(columnDomains.get().stream() .collect(toMap(ColumnDomain::getColumn, ColumnDomain::getDomain))); }
/** * Returns true only if the this TupleDomain contains all possible tuples that would be allowable by * the other TupleDomain. */ public boolean contains(TupleDomain<T> other) { return other.isNone() || columnWiseUnion(this, other).equals(this); }
@Test public void testGetUnenforcedPredicates() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains( ImmutableMap.of( col2, Domain.singleValue(BIGINT, 34L), col4, Domain.singleValue(BIGINT, 26L))); CassandraClusteringPredicatesExtractor predicatesExtractor = new CassandraClusteringPredicatesExtractor(cassandraTable.getClusteringKeyColumns(), tupleDomain, cassandraVersion); TupleDomain<ColumnHandle> unenforcedPredicates = TupleDomain.withColumnDomains(ImmutableMap.of(col4, Domain.singleValue(BIGINT, 26L))); assertEquals(predicatesExtractor.getUnenforcedConstraints(), unenforcedPredicates); } }
@Test public void testParquetTupleDomainStruct() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_struct", HiveType.valueOf("struct<a:int,b:int>"), parseTypeSignature(StandardTypes.ROW), 0, REGULAR, Optional.empty()); RowType.Field rowField = new RowType.Field(Optional.of("my_struct"), INTEGER); RowType rowType = RowType.from(ImmutableList.of(rowField)); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(rowType))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_struct", new PrimitiveType(OPTIONAL, INT32, "a"), new PrimitiveType(OPTIONAL, INT32, "b"))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
@Override public ActualProperties visitTableScan(TableScanNode node, List<ActualProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); ActualProperties.Builder properties = ActualProperties.builder(); // Globally constant assignments Map<ColumnHandle, NullableValue> globalConstants = new HashMap<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) .forEach(entry -> globalConstants.put(entry.getKey(), entry.getValue())); Map<Symbol, NullableValue> symbolConstants = globalConstants.entrySet().stream() .filter(entry -> assignments.containsKey(entry.getKey())) .collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue)); properties.constants(symbolConstants); // Partitioning properties properties.global(deriveGlobalProperties(layout, assignments, globalConstants)); // Append the global constants onto the local properties to maximize their translation potential List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties = ImmutableList.<LocalProperty<ColumnHandle>>builder() .addAll(globalConstants.keySet().stream().map(ConstantProperty::new).iterator()) .addAll(layout.getLocalProperties()) .build(); properties.local(LocalProperties.translate(constantAppendedLocalProperties, column -> Optional.ofNullable(assignments.get(column)))); return properties.build(); }
@Override public StreamProperties visitTableScan(TableScanNode node, List<StreamProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); // Globally constant assignments Set<ColumnHandle> constants = new HashSet<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) // TODO consider allowing nulls .forEach(entry -> constants.add(entry.getKey())); Optional<Set<Symbol>> streamPartitionSymbols = layout.getStreamPartitioningColumns() .flatMap(columns -> getNonConstantSymbols(columns, assignments, constants)); // if we are partitioned on empty set, we must say multiple of unknown partitioning, because // the connector does not guarantee a single split in this case (since it might not understand // that the value is a constant). if (streamPartitionSymbols.isPresent() && streamPartitionSymbols.get().isEmpty()) { return new StreamProperties(MULTIPLE, Optional.empty(), false); } return new StreamProperties(MULTIPLE, streamPartitionSymbols, false); }
public static Optional<HiveBucketFilter> getHiveBucketFilter(Table table, TupleDomain<ColumnHandle> effectivePredicate) if (!table.getStorage().getBucketProperty().isPresent()) { return Optional.empty(); Optional<Map<ColumnHandle, NullableValue>> bindings = TupleDomain.extractFixedValues(effectivePredicate); if (!bindings.isPresent()) { return Optional.empty(); if (!effectivePredicate.getDomains().isPresent()) { return Optional.empty(); Optional<Domain> domain = effectivePredicate.getDomains().get().entrySet().stream() .filter(entry -> ((HiveColumnHandle) entry.getKey()).getName().equals(BUCKET_COLUMN_NAME)) .findFirst() return Optional.empty(); ValueSet values = domain.get().getValues(); ImmutableSet.Builder<Integer> builder = ImmutableSet.builder(); int bucketCount = table.getStorage().getBucketProperty().get().getBucketCount();
private Map<TpchColumn<?>, List<Object>> getColumnValuesRestrictions(TpchTable<?> tpchTable, Constraint<ColumnHandle> constraint) { TupleDomain<ColumnHandle> constraintSummary = constraint.getSummary(); if (constraintSummary.isAll()) { return emptyMap(); } else if (constraintSummary.isNone()) { Set<TpchColumn<?>> columns = ImmutableSet.copyOf(tpchTable.getColumns()); return asMap(columns, key -> emptyList()); } else { Map<ColumnHandle, Domain> domains = constraintSummary.getDomains().get(); Optional<Domain> orderStatusDomain = Optional.ofNullable(domains.get(toColumnHandle(ORDER_STATUS))); Optional<Map<TpchColumn<?>, List<Object>>> allowedColumnValues = orderStatusDomain.map(domain -> { List<Object> allowedValues = ORDER_STATUS_VALUES.stream() .filter(domain::includesNullableValue) .collect(toList()); return avoidTrivialOrderStatusRestriction(allowedValues); }); return allowedColumnValues.orElse(emptyMap()); } }
if (!result.isPresent()) { return context.defaultRewrite(node); TableScanNode tableScan = result.get(); ImmutableMap.Builder<Symbol, Type> typesBuilder = ImmutableMap.builder(); ImmutableMap.Builder<Symbol, ColumnHandle> columnBuilder = ImmutableMap.builder(); if (!tableScan.getLayout().isPresent()) { List<TableLayoutResult> layouts = metadata.getLayouts(session, tableScan.getTable(), Constraint.alwaysTrue(), Optional.empty()); if (layouts.size() == 1) { ImmutableList.Builder<List<Expression>> rowsBuilder = ImmutableList.builder(); for (TupleDomain<ColumnHandle> domain : predicates.getPredicates()) { if (!domain.isNone()) { Map<ColumnHandle, NullableValue> entries = TupleDomain.extractFixedValues(domain).get(); ImmutableList.Builder<Expression> rowBuilder = ImmutableList.builder();
@Test public void testAddresses() { // split uses "example" scheme so no addresses are available and is not remotely accessible assertEquals(split.getAddresses(), ImmutableList.of()); assertEquals(split.isRemotelyAccessible(), true); JdbcSplit jdbcSplit = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", TupleDomain.all(), Optional.empty()); assertEquals(jdbcSplit.getAddresses(), ImmutableList.of()); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { JmxTableLayoutHandle jmxLayout = (JmxTableLayoutHandle) layout; JmxTableHandle tableHandle = jmxLayout.getTable(); TupleDomain<ColumnHandle> predicate = jmxLayout.getConstraint(); //TODO is there a better way to get the node column? Optional<JmxColumnHandle> nodeColumnHandle = tableHandle.getColumnHandles().stream() .filter(jmxColumnHandle -> jmxColumnHandle.getColumnName().equals(NODE_COLUMN_NAME)) .findFirst(); checkState(nodeColumnHandle.isPresent(), "Failed to find %s column", NODE_COLUMN_NAME); List<ConnectorSplit> splits = nodeManager.getAllNodes().stream() .filter(node -> { NullableValue value = NullableValue.of(createUnboundedVarcharType(), utf8Slice(node.getNodeIdentifier())); return predicate.overlaps(fromFixedValues(ImmutableMap.of(nodeColumnHandle.get(), value))); }) .map(node -> new JmxSplit(tableHandle, ImmutableList.of(node.getHostAndPort()))) .collect(toList()); return new FixedSplitSource(splits); } }
public static Optional<String> stringFilter(TupleDomain<Integer> constraint, int index) { if (constraint.isNone()) { return Optional.empty(); } Domain domain = constraint.getDomains().get().get(index); if ((domain == null) || !domain.isSingleValue()) { return Optional.empty(); } Object value = domain.getSingleValue(); if (value instanceof Slice) { return Optional.of(((Slice) value).toStringUtf8()); } return Optional.empty(); }
Map<ColumnHandle, NullableValue> fixedValues = TupleDomain.extractFixedValues(tpchIndexHandle.getFixedValues()).get(); checkArgument(lookupSchema.stream().noneMatch(handle -> fixedValues.keySet().contains(handle)), "Lookup columnHandles are not expected to overlap with the fixed value predicates"); List<ColumnHandle> fixedValueColumns = ImmutableList.copyOf(fixedValues.keySet()); List<ColumnHandle> finalLookupSchema = ImmutableList.<ColumnHandle>builder() .addAll(lookupSchema) .addAll(fixedValueColumns) checkState(indexedTable.isPresent()); TpchIndexedData.IndexedTable table = indexedTable.get();
@Test public void testParquetTupleDomainPrimitive() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_primitive", HiveType.valueOf("bigint"), parseTypeSignature(StandardTypes.BIGINT), 0, REGULAR, Optional.empty()); Domain singleValueDomain = Domain.singleValue(BIGINT, 123L); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, singleValueDomain)); MessageType fileSchema = new MessageType("hive_schema", new PrimitiveType(OPTIONAL, INT64, "my_primitive")); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertEquals(tupleDomain.getDomains().get().size(), 1); ColumnDescriptor descriptor = tupleDomain.getDomains().get().keySet().iterator().next(); assertEquals(descriptor.getPath().length, 1); assertEquals(descriptor.getPath()[0], "my_primitive"); Domain predicateDomain = Iterables.getOnlyElement(tupleDomain.getDomains().get().values()); assertEquals(predicateDomain, singleValueDomain); }
private static boolean isColumnPredicate(ColumnDescriptor columnDescriptor, TupleDomain<ColumnDescriptor> parquetTupleDomain) { verify(parquetTupleDomain.getDomains().isPresent(), "parquetTupleDomain is empty"); return parquetTupleDomain.getDomains().get().keySet().contains(columnDescriptor); }
@Override public ActualProperties visitFilter(FilterNode node, List<ActualProperties> inputProperties) { ActualProperties properties = Iterables.getOnlyElement(inputProperties); DomainTranslator.ExtractionResult decomposedPredicate = DomainTranslator.fromPredicate( metadata, session, node.getPredicate(), types); Map<Symbol, NullableValue> constants = new HashMap<>(properties.getConstants()); constants.putAll(extractFixedValues(decomposedPredicate.getTupleDomain()).orElse(ImmutableMap.of())); return ActualProperties.builderFrom(properties) .constants(constants) .build(); }