private static Optional<Domain> getDomain(OptionalInt timestampOrdinalPosition, TupleDomain<LocalFileColumnHandle> predicate) { Optional<Map<LocalFileColumnHandle, Domain>> domains = predicate.getDomains(); Domain domain = null; if (domains.isPresent() && timestampOrdinalPosition.isPresent()) { Map<LocalFileColumnHandle, Domain> domainMap = domains.get(); Set<Domain> timestampDomain = domainMap.entrySet().stream() .filter(entry -> entry.getKey().getOrdinalPosition() == timestampOrdinalPosition.getAsInt()) .map(Map.Entry::getValue) .collect(toSet()); if (!timestampDomain.isEmpty()) { domain = Iterables.getOnlyElement(timestampDomain); } } return Optional.ofNullable(domain); }
static List<Map<String, Object>> frames(final ClassLoader loader, final SourceLocator locator, final Throwable cause) { List<StackTraceElement> stacktrace = Arrays.asList(cause.getStackTrace()); int limit = IntStream.range(0, stacktrace.size()) .filter(i -> stacktrace.get(i).getClassName().equals(HANDLER)).findFirst() .orElse(stacktrace.size()); return stacktrace.stream() // trunk stack at HttpHandlerImpl (skip servers stack) .limit(limit) .map(e -> frame(loader, locator, cause, e)) .collect(Collectors.toList()); }
@Override public ScheduleResult schedule() { OptionalInt totalPartitions = OptionalInt.of(partitionToNode.size()); List<RemoteTask> newTasks = IntStream.range(0, partitionToNode.size()) .mapToObj(partition -> taskScheduler.scheduleTask(partitionToNode.get(partition), partition, totalPartitions)) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); return new ScheduleResult(true, newTasks, 0); } }
/** * If a value is present in {@code optional}, returns a stream containing only that element, * otherwise returns an empty stream. * * <p><b>Java 9 users:</b> use {@code optional.stream()} instead. */ public static IntStream stream(OptionalInt optional) { return optional.isPresent() ? IntStream.of(optional.getAsInt()) : IntStream.empty(); }
@Override public Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) { HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle; SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(table); if (!hiveBucketHandle.isPresent()) { return Optional.empty(); } HiveBucketProperty bucketProperty = table.getStorage().getBucketProperty() .orElseThrow(() -> new NoSuchElementException("Bucket property should be set")); if (!bucketProperty.getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) { throw new PrestoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled"); } HivePartitioningHandle partitioningHandle = new HivePartitioningHandle( hiveBucketHandle.get().getTableBucketCount(), hiveBucketHandle.get().getColumns().stream() .map(HiveColumnHandle::getHiveType) .collect(Collectors.toList()), OptionalInt.of(hiveBucketHandle.get().getTableBucketCount())); List<String> partitionColumns = hiveBucketHandle.get().getColumns().stream() .map(HiveColumnHandle::getName) .collect(Collectors.toList()); return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionColumns)); }
List<HivePartition> partitions = hiveLayoutHandle.getPartitions().get(); Optional<DiscretePredicates> discretePredicates = Optional.empty(); if (!partitionColumns.isEmpty()) { discretePredicates = Optional.of(new DiscretePredicates(partitionColumns, partitionDomains)); hiveBucketHandle.getReadBucketCount(), hiveBucketHandle.getColumns().stream() .map(HiveColumnHandle::getHiveType) .collect(Collectors.toList()), OptionalInt.empty()), hiveBucketHandle.getColumns().stream() .map(ColumnHandle.class::cast) .collect(toList()))); Optional.empty(), discretePredicates, ImmutableList.of());
.filter(TestColumn::isPartitionKey) .map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())) .collect(toList()); session, split.getPath(), OptionalInt.empty(), split.getStart(), split.getLength(), TYPE_MANAGER, ImmutableMap.of(), Optional.empty(), false); assertTrue(pageSource.isPresent()); checkPageSource(pageSource.get(), testColumns, getTypes(columnHandles), rowCount);
.filter(TestColumn::isPartitionKey) .map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())) .collect(toList()); session, split.getPath(), OptionalInt.empty(), split.getStart(), split.getLength(), TYPE_MANAGER, ImmutableMap.of(), Optional.empty(), false); RecordCursor cursor = ((RecordPageSource) pageSource.get()).getCursor();
public static Optional<HiveBucketFilter> getHiveBucketFilter(Table table, TupleDomain<ColumnHandle> effectivePredicate) if (!table.getStorage().getBucketProperty().isPresent()) { return Optional.empty(); if (!bindings.isPresent()) { return Optional.empty(); OptionalInt singleBucket = getHiveBucket(table, bindings.get()); if (singleBucket.isPresent()) { return Optional.of(new HiveBucketFilter(ImmutableSet.of(singleBucket.getAsInt()))); Optional<Domain> domain = effectivePredicate.getDomains().get().entrySet().stream() .filter(entry -> ((HiveColumnHandle) entry.getKey()).getName().equals(BUCKET_COLUMN_NAME)) .findFirst() .map(Entry::getValue); if (!domain.isPresent()) {
private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, File outputFile) { Properties splitProperties = new Properties(); splitProperties.setProperty(FILE_INPUT_FORMAT, config.getHiveStorageFormat().getInputFormat()); splitProperties.setProperty(SERIALIZATION_LIB, config.getHiveStorageFormat().getSerDe()); splitProperties.setProperty("columns", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getName).collect(toList()))); splitProperties.setProperty("columns.types", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getHiveType).map(hiveType -> hiveType.getHiveTypeName().toString()).collect(toList()))); HiveSplit split = new HiveSplit( SCHEMA_NAME, TABLE_NAME, "", "file:///" + outputFile.getAbsolutePath(), 0, outputFile.length(), outputFile.length(), splitProperties, ImmutableList.of(), ImmutableList.of(), OptionalInt.empty(), false, TupleDomain.all(), ImmutableMap.of(), Optional.empty(), false); HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER); return provider.createPageSource(transaction, getSession(config), split, ImmutableList.copyOf(getColumnHandles())); }
private OperatorFactory createLookupJoin( JoinNode node, PhysicalOperation probeSource, List<Symbol> probeSymbols, Optional<Symbol> probeHashSymbol, JoinBridgeManager<? extends LookupSourceFactory> lookupSourceFactoryManager, LocalExecutionPlanContext context) { List<Type> probeTypes = probeSource.getTypes(); List<Symbol> probeOutputSymbols = node.getOutputSymbols().stream() .filter(symbol -> node.getLeft().getOutputSymbols().contains(symbol)) .collect(toImmutableList()); List<Integer> probeOutputChannels = ImmutableList.copyOf(getChannelsForSymbols(probeOutputSymbols, probeSource.getLayout())); List<Integer> probeJoinChannels = ImmutableList.copyOf(getChannelsForSymbols(probeSymbols, probeSource.getLayout())); OptionalInt probeHashChannel = probeHashSymbol.map(channelGetter(probeSource)) .map(OptionalInt::of).orElse(OptionalInt.empty()); OptionalInt totalOperatorsCount = getJoinOperatorsCountForSpill(context, session); switch (node.getType()) { case INNER: return lookupJoinOperators.innerJoin(context.getNextOperatorId(), node.getId(), lookupSourceFactoryManager, probeTypes, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), totalOperatorsCount, partitioningSpillerFactory); case LEFT: return lookupJoinOperators.probeOuterJoin(context.getNextOperatorId(), node.getId(), lookupSourceFactoryManager, probeTypes, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), totalOperatorsCount, partitioningSpillerFactory); case RIGHT: return lookupJoinOperators.lookupOuterJoin(context.getNextOperatorId(), node.getId(), lookupSourceFactoryManager, probeTypes, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), totalOperatorsCount, partitioningSpillerFactory); case FULL: return lookupJoinOperators.fullOuterJoin(context.getNextOperatorId(), node.getId(), lookupSourceFactoryManager, probeTypes, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), totalOperatorsCount, partitioningSpillerFactory); default: throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); } }
public StreamingAggregationOperator(OperatorContext operatorContext, List<Type> sourceTypes, List<Type> groupByTypes, List<Integer> groupByChannels, Step step, List<AccumulatorFactory> accumulatorFactories, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(StreamingAggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = Ints.toArray(requireNonNull(groupByChannels, "groupByChannels is null")); this.accumulatorFactories = requireNonNull(accumulatorFactories, "accumulatorFactories is null"); this.step = requireNonNull(step, "step is null"); this.aggregates = setupAggregates(step, accumulatorFactories); this.pageBuilder = new PageBuilder(toTypes(groupByTypes, aggregates)); requireNonNull(joinCompiler, "joinCompiler is null"); requireNonNull(sourceTypes, "sourceTypes is null"); pagesHashStrategy = joinCompiler.compilePagesHashStrategyFactory(sourceTypes, groupByChannels, Optional.empty()) .createPagesHashStrategy( sourceTypes.stream() .map(type -> ImmutableList.<Block>of()) .collect(toImmutableList()), OptionalInt.empty()); }
.map(ImmutableSet::copyOf) .collect(toList()); .mapToInt(Set::size) .sum(); newBaseExpressions = Math.multiplyExact(subPredicates.stream() .mapToInt(Set::size) .reduce(Math::multiplyExact) .getAsInt(), subPredicates.size()); crossProduct.stream() .map(expressions -> combinePredicates(expression.getOperator(), expressions)) .collect(toImmutableList()));
private static void addLookupOuterDrivers(LocalExecutionPlanContext context) { // For an outer join on the lookup side (RIGHT or FULL) add an additional // driver to output the unused rows in the lookup source for (DriverFactory factory : context.getDriverFactories()) { List<OperatorFactory> operatorFactories = factory.getOperatorFactories(); for (int i = 0; i < operatorFactories.size(); i++) { OperatorFactory operatorFactory = operatorFactories.get(i); if (!(operatorFactory instanceof JoinOperatorFactory)) { continue; } JoinOperatorFactory lookupJoin = (JoinOperatorFactory) operatorFactory; Optional<OuterOperatorFactoryResult> outerOperatorFactoryResult = lookupJoin.createOuterOperatorFactory(); if (outerOperatorFactoryResult.isPresent()) { // Add a new driver to output the unmatched rows in an outer join. // We duplicate all of the factories above the JoinOperator (the ones reading from the joins), // and replace the JoinOperator with the OuterOperator (the one that produces unmatched rows). ImmutableList.Builder<OperatorFactory> newOperators = ImmutableList.builder(); newOperators.add(outerOperatorFactoryResult.get().getOuterOperatorFactory()); operatorFactories.subList(i + 1, operatorFactories.size()).stream() .map(OperatorFactory::duplicate) .forEach(newOperators::add); context.addDriverFactory(false, factory.isOutputDriver(), newOperators.build(), OptionalInt.of(1), outerOperatorFactoryResult.get().getBuildExecutionStrategy()); } } } }
private static void assertData(LocalFileTables localFileTables, LocalFileMetadata metadata) List<LocalFileColumnHandle> columnHandles = metadata.getColumnHandles(SESSION, new LocalFileTableHandle(tableName, OptionalInt.of(0), OptionalInt.of(-1))) .values().stream().map(column -> (LocalFileColumnHandle) column) .collect(Collectors.toList()); assertEquals(cursor.getType(i), columnHandles.get(i).getColumnType()); assertTrue(cursor.advanceNextPosition()); assertEquals(cursor.getSlice(0).toStringUtf8(), address.toString()); assertEquals(cursor.getSlice(2).toStringUtf8(), "127.0.0.1"); assertEquals(cursor.getSlice(3).toStringUtf8(), "POST");
@Test public void testIncreasingBatchSize() { int rows = 1024; // We deliberately do not set the ticker, so that the expression is always cheap and the batch size gets doubled until other limits are hit TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(1), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // skip the first read && skip the last read, which can be a partial page if (positionCount > 1 && totalPositionCount != rows) { assertEquals(positionCount, previousPositionCount * 2); } previousPositionCount = positionCount; } }
@Test public void testSelectAllFilter() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(MAX_BATCH_SIZE)); Page inputPage = new Page(createLongSequenceBlock(0, 100)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 1); assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100))); }