public TpchIndexedData(String connectorId, TpchIndexSpec tpchIndexSpec) { requireNonNull(connectorId, "connectorId is null"); requireNonNull(tpchIndexSpec, "tpchIndexSpec is null"); TpchMetadata tpchMetadata = new TpchMetadata(connectorId); TpchRecordSetProvider tpchRecordSetProvider = new TpchRecordSetProvider(); ImmutableMap.Builder<Set<TpchScaledColumn>, IndexedTable> indexedTablesBuilder = ImmutableMap.builder(); Set<TpchScaledTable> tables = tpchIndexSpec.listIndexedTables(); for (TpchScaledTable table : tables) { SchemaTableName tableName = new SchemaTableName("sf" + table.getScaleFactor(), table.getTableName()); TpchTableHandle tableHandle = tpchMetadata.getTableHandle(null, tableName); Map<String, ColumnHandle> columnHandles = new LinkedHashMap<>(tpchMetadata.getColumnHandles(null, tableHandle)); for (Set<String> columnNames : tpchIndexSpec.getColumnIndexes(table)) { List<String> keyColumnNames = ImmutableList.copyOf(columnNames); // Finalize the key order Set<TpchScaledColumn> keyColumns = keyColumnNames.stream() .map(name -> new TpchScaledColumn(table, name)) .collect(toImmutableSet()); TpchTable<?> tpchTable = TpchTable.getTable(table.getTableName()); RecordSet recordSet = tpchRecordSetProvider.getRecordSet(tpchTable, ImmutableList.copyOf(columnHandles.values()), table.getScaleFactor(), 0, 1, TupleDomain.all()); IndexedTable indexedTable = indexTable(recordSet, ImmutableList.copyOf(columnHandles.keySet()), keyColumnNames); indexedTablesBuilder.put(keyColumns, indexedTable); } } indexedTables = indexedTablesBuilder.build(); }
private void insertRows(TpchMetadata tpchMetadata, TpchTable tpchTable) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(null, new SchemaTableName(TINY_SCHEMA_NAME, tpchTable.getTableName())); insertRows(tpchMetadata.getTableMetadata(null, tableHandle), handle, createTpchRecordSet(tpchTable, tableHandle.getScaleFactor())); }
private void testTableStats(String schema, TpchTable<?> table, Constraint<ColumnHandle> constraint, double expectedRowCount) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, constraint); double actualRowCountValue = tableStatistics.getRowCount().getValue(); assertEquals(tableStatistics.getRowCount(), Estimate.of(actualRowCountValue)); assertEquals(actualRowCountValue, expectedRowCount, expectedRowCount * TOLERANCE); }
private void testNoTableStats(String schema, TpchTable<?> table) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, alwaysTrue()); assertTrue(tableStatistics.getRowCount().isUnknown()); }
private void testColumnStats(String schema, TpchTable<?> table, TpchColumn<?> column, Constraint<ColumnHandle> constraint, ColumnStatistics expected) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, constraint); ColumnHandle columnHandle = tpchMetadata.getColumnHandles(session, tableHandle).get(column.getSimplifiedColumnName()); ColumnStatistics actual = tableStatistics.getColumnStatistics().get(columnHandle); EstimateAssertion estimateAssertion = new EstimateAssertion(TOLERANCE); estimateAssertion.assertClose(actual.getDistinctValuesCount(), expected.getDistinctValuesCount(), "distinctValuesCount"); estimateAssertion.assertClose(actual.getDataSize(), expected.getDataSize(), "dataSize"); estimateAssertion.assertClose(actual.getNullsFraction(), expected.getNullsFraction(), "nullsFraction"); estimateAssertion.assertClose(actual.getRange(), expected.getRange(), "range"); }
@Test public void testOrdersOrderStatusPredicatePushdown() { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", ORDERS.getTableName())); TupleDomain<ColumnHandle> domain; ConnectorTableLayoutResult tableLayout; domain = fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P")); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); domain = fixedValueTupleDomain(tpchMetadata, ORDER_KEY, 42L); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), domain, session); assertTupleDomainEquals( tableLayout.getTableLayout().getPredicate(), // The most important thing about the expected value that it is NOT TupleDomain.none() (or equivalent). // Using concrete expected value instead of checking TupleDomain::isNone to make sure the test doesn't pass on some other wrong value. TupleDomain.columnWiseUnion( fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("F")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("O")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P"))), session); }
@Test public void testPartTypeAndPartContainerPredicatePushdown() TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", PART.getTableName()));
public TpchIndexedData(String connectorId, TpchIndexSpec tpchIndexSpec) { requireNonNull(connectorId, "connectorId is null"); requireNonNull(tpchIndexSpec, "tpchIndexSpec is null"); TpchMetadata tpchMetadata = new TpchMetadata(connectorId); TpchRecordSetProvider tpchRecordSetProvider = new TpchRecordSetProvider(); ImmutableMap.Builder<Set<TpchScaledColumn>, IndexedTable> indexedTablesBuilder = ImmutableMap.builder(); Set<TpchScaledTable> tables = tpchIndexSpec.listIndexedTables(); for (TpchScaledTable table : tables) { SchemaTableName tableName = new SchemaTableName("sf" + table.getScaleFactor(), table.getTableName()); TpchTableHandle tableHandle = tpchMetadata.getTableHandle(null, tableName); Map<String, ColumnHandle> columnHandles = new LinkedHashMap<>(tpchMetadata.getColumnHandles(null, tableHandle)); for (Set<String> columnNames : tpchIndexSpec.getColumnIndexes(table)) { List<String> keyColumnNames = ImmutableList.copyOf(columnNames); // Finalize the key order Set<TpchScaledColumn> keyColumns = keyColumnNames.stream() .map(name -> new TpchScaledColumn(table, name)) .collect(toImmutableSet()); TpchTable<?> tpchTable = TpchTable.getTable(table.getTableName()); RecordSet recordSet = tpchRecordSetProvider.getRecordSet(tpchTable, ImmutableList.copyOf(columnHandles.values()), table.getScaleFactor(), 0, 1, TupleDomain.all()); IndexedTable indexedTable = indexTable(recordSet, ImmutableList.copyOf(columnHandles.keySet()), keyColumnNames); indexedTablesBuilder.put(keyColumns, indexedTable); } } indexedTables = indexedTablesBuilder.build(); }
private void insertRows(TpchMetadata tpchMetadata, TpchTable tpchTable) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(null, new SchemaTableName(TINY_SCHEMA_NAME, tpchTable.getTableName())); insertRows(tpchMetadata.getTableMetadata(null, tableHandle), handle, createTpchRecordSet(tpchTable, tableHandle.getScaleFactor())); }
private void testTableStats(String schema, TpchTable<?> table, Constraint<ColumnHandle> constraint, double expectedRowCount) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, constraint); double actualRowCountValue = tableStatistics.getRowCount().getValue(); assertEquals(tableStatistics.getRowCount(), Estimate.of(actualRowCountValue)); assertEquals(actualRowCountValue, expectedRowCount, expectedRowCount * TOLERANCE); }
private void testNoTableStats(String schema, TpchTable<?> table) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, alwaysTrue()); assertTrue(tableStatistics.getRowCount().isUnknown()); }
private void testColumnStats(String schema, TpchTable<?> table, TpchColumn<?> column, Constraint<ColumnHandle> constraint, ColumnStatistics expected) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, constraint); ColumnHandle columnHandle = tpchMetadata.getColumnHandles(session, tableHandle).get(column.getSimplifiedColumnName()); ColumnStatistics actual = tableStatistics.getColumnStatistics().get(columnHandle); EstimateAssertion estimateAssertion = new EstimateAssertion(TOLERANCE); estimateAssertion.assertClose(actual.getDistinctValuesCount(), expected.getDistinctValuesCount(), "distinctValuesCount"); estimateAssertion.assertClose(actual.getDataSize(), expected.getDataSize(), "dataSize"); estimateAssertion.assertClose(actual.getNullsFraction(), expected.getNullsFraction(), "nullsFraction"); estimateAssertion.assertClose(actual.getRange(), expected.getRange(), "range"); }
@Test public void testOrdersOrderStatusPredicatePushdown() { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", ORDERS.getTableName())); TupleDomain<ColumnHandle> domain; ConnectorTableLayoutResult tableLayout; domain = fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P")); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); domain = fixedValueTupleDomain(tpchMetadata, ORDER_KEY, 42L); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), domain, session); assertTupleDomainEquals( tableLayout.getTableLayout().getPredicate(), // The most important thing about the expected value that it is NOT TupleDomain.none() (or equivalent). // Using concrete expected value instead of checking TupleDomain::isNone to make sure the test doesn't pass on some other wrong value. TupleDomain.columnWiseUnion( fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("F")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("O")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P"))), session); }
@Test public void testPartTypeAndPartContainerPredicatePushdown() TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", PART.getTableName()));