private WorkProcessor<Page> mergeFromDisk() { checkState(spiller.isPresent()); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList()), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, memoryLimitForMerge); }
private WorkProcessor<Page> mergeFromDiskAndMemory() { checkState(spiller.isPresent()); hashAggregationBuilder.setOutputPartial(); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), ImmutableList.<WorkProcessor<Page>>builder() .addAll(spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList())) .add(hashAggregationBuilder.buildHashSortedResult()) .build(), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, max(memoryLimitForMerge - memoryLimitForMergeWithMemory, 1L)); }
private static AggregatedMemoryContext mockMemoryContext(ScheduledExecutorService scheduledExecutor) { // It's important to use OperatorContext's system memory context, because it does additional bookkeeping. return TestingOperatorContext.create(scheduledExecutor).newAggregateSystemMemoryContext(); }
private Page spillAndMaskSpilledPositions(Page page, IntPredicate spillMask) { checkState(spillInProgress.isDone(), "Previous spill still in progress"); checkSuccess(spillInProgress, "spilling failed"); if (!spiller.isPresent()) { spiller = Optional.of(partitioningSpillerFactory.create( probeTypes, getPartitionGenerator(), operatorContext.getSpillContext().newLocalSpillContext(), operatorContext.newAggregateSystemMemoryContext())); } PartitioningSpillResult result = spiller.get().partitionAndSpill(page, spillMask); spillInProgress = result.getSpillingFuture(); return result.getRetained(); }
private ListenableFuture<?> spillToDisk() { checkState(hasPreviousSpillCompletedSuccessfully(), "Previous spill hasn't yet finished"); hashAggregationBuilder.setOutputPartial(); if (!spiller.isPresent()) { spiller = Optional.of(spillerFactory.create( hashAggregationBuilder.buildTypes(), operatorContext.getSpillContext(), operatorContext.newAggregateSystemMemoryContext())); } // start spilling process with current content of the hashAggregationBuilder builder... spillInProgress = spiller.get().spill(hashAggregationBuilder.buildHashSortedResult().iterator()); // ... and immediately create new hashAggregationBuilder so effectively memory ownership // over hashAggregationBuilder is transferred from this thread to a spilling thread rebuildHashAggregationBuilder(); return spillInProgress; }