private Optional<String> getPartitionDescription(Optional<TpchColumn<?>> partitionColumn, Optional<String> partitionValue) { checkArgument(partitionColumn.isPresent() == partitionValue.isPresent()); return withBoth(partitionColumn, partitionValue, (column, value) -> column.getColumnName() + "." + value); } }
private Optional<Long> addDistinctValuesCount(TpchColumn<?> partitionColumn, String columnName, ColumnStatisticsData leftStats, ColumnStatisticsData rightStats) { //unique values count can't be added between different partitions //for columns other than the partition column (because almost certainly there are duplicates) return combine(leftStats.getDistinctValuesCount(), rightStats.getDistinctValuesCount(), (a, b) -> a + b) .filter(v -> columnName.equals(partitionColumn.getColumnName())); }
private <E extends TpchEntity> TableStatisticsData recordStatistics(Iterable<E> rows, List<TpchColumn<E>> columns, Predicate<E> constraint) { Map<TpchColumn<E>, ColumnStatisticsRecorder> statisticsRecorders = createStatisticsRecorders(columns); long rowCount = 0; for (E row : rows) { if (constraint.test(row)) { rowCount++; for (TpchColumn<E> column : columns) { Comparable<?> value = getTpchValue(row, column); statisticsRecorders.get(column).record(value); } } } Map<String, ColumnStatisticsData> columnSampleStatistics = statisticsRecorders.entrySet().stream() .collect(toImmutableMap( e -> e.getKey().getColumnName(), e -> e.getValue().getRecording())); return new TableStatisticsData(rowCount, columnSampleStatistics); }
private Optional<String> getPartitionDescription(Optional<TpchColumn<?>> partitionColumn, Optional<String> partitionValue) { checkArgument(partitionColumn.isPresent() == partitionValue.isPresent()); return withBoth(partitionColumn, partitionValue, (column, value) -> column.getColumnName() + "." + value); } }
private Optional<String> getPartitionDescription(Optional<TpchColumn<?>> partitionColumn, Optional<String> partitionValue) { checkArgument(partitionColumn.isPresent() == partitionValue.isPresent()); return withBoth(partitionColumn, partitionValue, (column, value) -> column.getColumnName() + "." + value); } }
private Optional<String> getPartitionDescription(Optional<TpchColumn<?>> partitionColumn, Optional<String> partitionValue) { checkArgument(partitionColumn.isPresent() == partitionValue.isPresent()); return withBoth(partitionColumn, partitionValue, (column, value) -> column.getColumnName() + "." + value); } }
private Optional<Long> addDistinctValuesCount(TpchColumn<?> partitionColumn, String columnName, ColumnStatisticsData leftStats, ColumnStatisticsData rightStats) { //unique values count can't be added between different partitions //for columns other than the partition column (because almost certainly there are duplicates) return combine(leftStats.getDistinctValuesCount(), rightStats.getDistinctValuesCount(), (a, b) -> a + b) .filter(v -> columnName.equals(partitionColumn.getColumnName())); }
private Optional<Long> addDistinctValuesCount(TpchColumn<?> partitionColumn, String columnName, ColumnStatisticsData leftStats, ColumnStatisticsData rightStats) { //unique values count can't be added between different partitions //for columns other than the partition column (because almost certainly there are duplicates) return combine(leftStats.getDistinctValuesCount(), rightStats.getDistinctValuesCount(), (a, b) -> a + b) .filter(v -> columnName.equals(partitionColumn.getColumnName())); }
private Optional<Long> addDistinctValuesCount(TpchColumn<?> partitionColumn, String columnName, ColumnStatisticsData leftStats, ColumnStatisticsData rightStats) { //unique values count can't be added between different partitions //for columns other than the partition column (because almost certainly there are duplicates) return combine(leftStats.getDistinctValuesCount(), rightStats.getDistinctValuesCount(), (a, b) -> a + b) .filter(v -> columnName.equals(partitionColumn.getColumnName())); }
private static ConnectorTableMetadata getTableMetadata(String schemaName, TpchTable<?> tpchTable) { ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); for (TpchColumn<? extends TpchEntity> column : tpchTable.getColumns()) { columns.add(new ColumnMetadata(column.getColumnName(), getPrestoType(column.getType()))); } columns.add(new ColumnMetadata(ROW_NUMBER_COLUMN_NAME, BIGINT, null, true)); SchemaTableName tableName = new SchemaTableName(schemaName, tpchTable.getTableName()); return new ConnectorTableMetadata(tableName, columns.build()); }
@Test public void testAllColumnNamesHaveValidPrefixes() { forEachTpchColumn(column -> { String columnName = column.getColumnName(); assertTrue(hasValidColumnPrefix(columnName), format("Invalid column name prefix for column %s (%s)", column, columnName)); }); }
@Test public void testSimplifiedColumnNamesAreSuffixesOfColumnNames() { forEachTpchColumn(column -> assertEquals(column.getSimplifiedColumnName(), stripPrefix(column.getColumnName()))); }
@Test public void testGetColumn() { TpchTable.getTables().forEach(table -> { table.getColumns().forEach(column -> { assertEquals(table.getColumn(column.getColumnName()), column); assertEquals(table.getColumn(column.getSimplifiedColumnName()), column); }); }); } }
private <E extends TpchEntity> TableStatisticsData recordStatistics(Iterable<E> rows, List<TpchColumn<E>> columns, Predicate<E> constraint) { Map<TpchColumn<E>, ColumnStatisticsRecorder> statisticsRecorders = createStatisticsRecorders(columns); long rowCount = 0; for (E row : rows) { if (constraint.test(row)) { rowCount++; for (TpchColumn<E> column : columns) { Comparable<?> value = getTpchValue(row, column); statisticsRecorders.get(column).record(value); } } } Map<String, ColumnStatisticsData> columnSampleStatistics = statisticsRecorders.entrySet().stream() .collect(toImmutableMap( e -> e.getKey().getColumnName(), e -> e.getValue().getRecording())); return new TableStatisticsData(rowCount, columnSampleStatistics); }
private <E extends TpchEntity> TableStatisticsData recordStatistics(Iterable<E> rows, List<TpchColumn<E>> columns, Predicate<E> constraint) { Map<TpchColumn<E>, ColumnStatisticsRecorder> statisticsRecorders = createStatisticsRecorders(columns); long rowCount = 0; for (E row : rows) { if (constraint.test(row)) { rowCount++; for (TpchColumn<E> column : columns) { Comparable<?> value = getTpchValue(row, column); statisticsRecorders.get(column).record(value); } } } Map<String, ColumnStatisticsData> columnSampleStatistics = statisticsRecorders.entrySet().stream() .collect(toImmutableMap( e -> e.getKey().getColumnName(), e -> e.getValue().getRecording())); return new TableStatisticsData(rowCount, columnSampleStatistics); }
private <E extends TpchEntity> TableStatisticsData recordStatistics(Iterable<E> rows, List<TpchColumn<E>> columns, Predicate<E> constraint) { Map<TpchColumn<E>, ColumnStatisticsRecorder> statisticsRecorders = createStatisticsRecorders(columns); long rowCount = 0; for (E row : rows) { if (constraint.test(row)) { rowCount++; for (TpchColumn<E> column : columns) { Comparable<?> value = getTpchValue(row, column); statisticsRecorders.get(column).record(value); } } } Map<String, ColumnStatisticsData> columnSampleStatistics = statisticsRecorders.entrySet().stream() .collect(toImmutableMap( e -> e.getKey().getColumnName(), e -> e.getValue().getRecording())); return new TableStatisticsData(rowCount, columnSampleStatistics); }