private static TupleDomain<HiveColumnHandle> toCompactTupleDomain(TupleDomain<ColumnHandle> effectivePredicate, int threshold) { ImmutableMap.Builder<HiveColumnHandle, Domain> builder = ImmutableMap.builder(); effectivePredicate.getDomains().ifPresent(domains -> { for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) { HiveColumnHandle hiveColumnHandle = (HiveColumnHandle) entry.getKey(); ValueSet values = entry.getValue().getValues(); ValueSet compactValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform( ranges -> ranges.getRangeCount() > threshold ? Optional.of(ValueSet.ofRanges(ranges.getSpan())) : Optional.empty(), discreteValues -> discreteValues.getValues().size() > threshold ? Optional.of(ValueSet.all(values.getType())) : Optional.empty(), allOrNone -> Optional.empty()) .orElse(values); builder.put(hiveColumnHandle, Domain.create(compactValueSet, entry.getValue().isNullAllowed())); } }); return TupleDomain.withColumnDomains(builder.build()); }
Map<ColumnHandle, NullableValue> fixedValues = TupleDomain.extractFixedValues(tupleDomain).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !indexableColumns.contains(entry.getKey())) filteredTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(tupleDomain.getDomains().get(), not(in(fixedValues.keySet()))));
@Test public void testBuildQueryOr() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( COL1, Domain.create(ValueSet.ofRanges(lessThan(BIGINT, 100L), greaterThan(BIGINT, 200L)), false))); Document query = MongoSession.buildQuery(tupleDomain); Document expected = new Document("$or", asList( new Document(COL1.getName(), new Document("$lt", 100L)), new Document(COL1.getName(), new Document("$gt", 200L)))); assertEquals(query, expected); }
@Test public void testIsAll() { assertTrue(TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of()).isAll()); assertFalse(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.singleValue(BIGINT, 0L))).isAll()); assertTrue(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.all(BIGINT))).isAll()); assertFalse(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.singleValue(BIGINT, 0L), B, Domain.all(BIGINT))).isAll()); }
@Test public void testToPredicateNone() { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(C_BIGINT, Domain.singleValue(BIGINT, 1L)) .put(C_DOUBLE, Domain.onlyNull(DOUBLE)) .put(C_VARCHAR, Domain.notNull(VARCHAR)) .put(C_BOOLEAN, Domain.none(BOOLEAN)) .build()); assertEquals(toPredicate(tupleDomain), FALSE_LITERAL); }
private void testSimpleComparison(Expression expression, Symbol symbol, Domain domain) { ExtractionResult result = fromPredicate(expression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); TupleDomain<Symbol> actual = result.getTupleDomain(); TupleDomain<Symbol> expected = withColumnDomains(ImmutableMap.of(symbol, domain)); if (!actual.equals(expected)) { fail(format("for comparison [%s] expected %s but found %s", expression.toString(), expected.toString(SESSION), actual.toString(SESSION))); } }
invalidTable, ImmutableList.of(), ImmutableList.of(new HivePartition(invalidTable, "unknown", ImmutableMap.of())), TupleDomain.all(), TupleDomain.all(), .build(); partitionCount = partitions.size(); tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2012-12-29")))); tableLayout = new ConnectorTableLayout( new HiveTableLayoutHandle(tablePartitionFormat, partitionColumns, partitions, tupleDomain, tupleDomain, Optional.empty(), Optional.empty()), Optional.empty(), TupleDomain.withColumnDomains(ImmutableMap.of( dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("textfile")), Range.equal(createUnboundedVarcharType(), utf8Slice("sequencefile")), Range.equal(createUnboundedVarcharType(), utf8Slice("rctext")), Range.equal(createUnboundedVarcharType(), utf8Slice("rcbinary"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 1L), Range.equal(INTEGER, 2L), Range.equal(INTEGER, 3L), Range.equal(INTEGER, 4L)), false))), Optional.empty(), Optional.empty(), Optional.of(new DiscretePredicates(partitionColumns, ImmutableList.of( TupleDomain.withColumnDomains(ImmutableMap.of( TupleDomain.withColumnDomains(ImmutableMap.of( TupleDomain.withColumnDomains(ImmutableMap.of( TupleDomain.withColumnDomains(ImmutableMap.of(
@Test(expectedExceptions = IllegalArgumentException.class) public void testTransformFailsWithNonUniqueMapping() { Map<Integer, Domain> domains = ImmutableMap.<Integer, Domain>builder() .put(1, Domain.singleValue(BIGINT, 1L)) .put(2, Domain.singleValue(BIGINT, 2L)) .put(3, Domain.singleValue(BIGINT, 3L)) .build(); TupleDomain<Integer> domain = TupleDomain.withColumnDomains(domains); domain.transform(input -> "x"); }
@Test public void testRoundTrip() { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(C_BIGINT, Domain.singleValue(BIGINT, 1L)) .put(C_DOUBLE, Domain.onlyNull(DOUBLE)) .put(C_VARCHAR, Domain.notNull(VARCHAR)) .put(C_BOOLEAN, Domain.singleValue(BOOLEAN, true)) .put(C_BIGINT_1, Domain.singleValue(BIGINT, 2L)) .put(C_DOUBLE_1, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(DOUBLE, 1.1), Range.equal(DOUBLE, 2.0), Range.range(DOUBLE, 3.0, false, 3.5, true)), true)) .put(C_VARCHAR_1, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("2013-01-01")), Range.greaterThan(VARCHAR, utf8Slice("2013-10-01"))), false)) .put(C_TIMESTAMP, Domain.singleValue(TIMESTAMP, TIMESTAMP_VALUE)) .put(C_DATE, Domain.singleValue(DATE, DATE_VALUE)) .put(C_COLOR, Domain.singleValue(COLOR, COLOR_VALUE_1)) .put(C_HYPER_LOG_LOG, Domain.notNull(HYPER_LOG_LOG)) .build()); assertPredicateTranslates(toPredicate(tupleDomain), tupleDomain); }
@Test public void testBuildQueryNull() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( COL1, Domain.create(ValueSet.ofRanges(greaterThan(BIGINT, 200L)), true))); Document query = MongoSession.buildQuery(tupleDomain); Document expected = new Document("$or", asList( new Document(COL1.getName(), new Document("$gt", 200L)), new Document(COL1.getName(), new Document("$exists", true).append("$eq", null)))); assertEquals(query, expected); } }
@Test public void testIsNone() { assertFalse(TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of()).isNone()); assertFalse(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.singleValue(BIGINT, 0L))).isNone()); assertTrue(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.none(BIGINT))).isNone()); assertFalse(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.all(BIGINT))).isNone()); assertTrue(TupleDomain.withColumnDomains(ImmutableMap.of(A, Domain.all(BIGINT), B, Domain.none(BIGINT))).isNone()); }
@Test public void testTransform() { Map<Integer, Domain> domains = ImmutableMap.<Integer, Domain>builder() .put(1, Domain.singleValue(BIGINT, 1L)) .put(2, Domain.singleValue(BIGINT, 2L)) .put(3, Domain.singleValue(BIGINT, 3L)) .build(); TupleDomain<Integer> domain = TupleDomain.withColumnDomains(domains); TupleDomain<String> transformed = domain.transform(Object::toString); Map<String, Domain> expected = ImmutableMap.<String, Domain>builder() .put("1", Domain.singleValue(BIGINT, 1L)) .put("2", Domain.singleValue(BIGINT, 2L)) .put("3", Domain.singleValue(BIGINT, 3L)) .build(); assertEquals(transformed.getDomains().get(), expected); }
Domain domain = complementIfNecessary(Domain.notNull(type), complement); return new ExtractionResult( TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), TRUE_LITERAL); TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), TRUE_LITERAL);
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
TupleDomain<ColumnHandle> tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2015-07-03")))); Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); TupleDomain<ColumnHandle> tupleDomain2 = TupleDomain.withColumnDomains( ImmutableMap.of(dsColumnHandle, Domain.create(ValueSet.ofRanges(Range.range(createUnboundedVarcharType(), utf8Slice("2015-07-01"), true, utf8Slice("2015-07-02"), true)), false))); Constraint<ColumnHandle> constraint2 = new Constraint<>(tupleDomain2, convertToPredicate(tupleDomain2)); List<ConnectorTableLayoutResult> tableLayoutResults2 = metadata.getTableLayouts(session, tableHandle, constraint2, Optional.empty());
private static TupleDomain<HiveColumnHandle> toCompactTupleDomain(TupleDomain<ColumnHandle> effectivePredicate, int threshold) { checkArgument(effectivePredicate.getDomains().isPresent()); ImmutableMap.Builder<HiveColumnHandle, Domain> builder = ImmutableMap.builder(); for (Map.Entry<ColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) { HiveColumnHandle hiveColumnHandle = checkType(entry.getKey(), HiveColumnHandle.class, "ConnectorColumnHandle"); ValueSet values = entry.getValue().getValues(); ValueSet compactValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform( ranges -> ranges.getRangeCount() > threshold ? Optional.of(ValueSet.ofRanges(ranges.getSpan())) : Optional.empty(), discreteValues -> discreteValues.getValues().size() > threshold ? Optional.of(ValueSet.all(values.getType())) : Optional.empty(), allOrNone -> Optional.empty()) .orElse(values); builder.put(hiveColumnHandle, Domain.create(compactValueSet, entry.getValue().isNullAllowed())); } return TupleDomain.withColumnDomains(builder.build()); }
@Test public void testAll() { assertTrue(TupleDomain.all().isAll()); assertEquals(TupleDomain.<ColumnHandle>all(), TupleDomain.withColumnDomains(ImmutableMap.of( A, Domain.all(BIGINT)))); assertEquals(TupleDomain.<ColumnHandle>all(), TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of())); }
@Test public void testFromIsNullPredicate() { assertPredicateTranslates( isNull(C_BIGINT), withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.onlyNull(BIGINT)))); assertPredicateTranslates( isNull(C_HYPER_LOG_LOG), withColumnDomains(ImmutableMap.of(C_HYPER_LOG_LOG, Domain.onlyNull(HYPER_LOG_LOG)))); assertPredicateTranslates( not(isNull(C_BIGINT)), withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.notNull(BIGINT)))); assertPredicateTranslates( not(isNull(C_HYPER_LOG_LOG)), withColumnDomains(ImmutableMap.of(C_HYPER_LOG_LOG, Domain.notNull(HYPER_LOG_LOG)))); }
@Test public void testToPredicateAllIgnored() { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(C_BIGINT, Domain.singleValue(BIGINT, 1L)) .put(C_DOUBLE, Domain.onlyNull(DOUBLE)) .put(C_VARCHAR, Domain.notNull(VARCHAR)) .put(C_BOOLEAN, Domain.all(BOOLEAN)) .build()); ExtractionResult result = fromPredicate(toPredicate(tupleDomain)); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(C_BIGINT, Domain.singleValue(BIGINT, 1L)) .put(C_DOUBLE, Domain.onlyNull(DOUBLE)) .put(C_VARCHAR, Domain.notNull(VARCHAR)) .build())); }
@Override protected ExtractionResult visitIsNullPredicate(IsNullPredicate node, Boolean complement) { if (!(node.getValue() instanceof SymbolReference)) { return super.visitIsNullPredicate(node, complement); } Symbol symbol = Symbol.from(node.getValue()); Type columnType = checkedTypeLookup(symbol); Domain domain = complementIfNecessary(Domain.onlyNull(columnType), complement); return new ExtractionResult( TupleDomain.withColumnDomains(ImmutableMap.of(symbol, domain)), TRUE_LITERAL); }