@Test public void testEmptyInput() { GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder( ImmutableList.of(BIGINT), (left, leftPosition, right, rightPosition) -> { throw new UnsupportedOperationException(); }, 5, false, new NoChannelGroupByHash()); assertFalse(groupedTopNBuilder.buildResult().hasNext()); }
private void updateMemoryReservation() { localUserMemoryContext.setBytes(topNBuilder.getEstimatedSizeInBytes()); }
@Override public void addInput(Page page) { checkState(!finishing, "Operator is already finishing"); checkState(unfinishedWork == null, "Cannot add input with the operator when unfinished work is not empty"); checkState(outputIterator == null, "Cannot add input with the operator when flushing"); requireNonNull(page, "page is null"); unfinishedWork = groupedTopNBuilder.processPage(page); if (unfinishedWork.process()) { unfinishedWork = null; } updateMemoryReservation(); }
GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), assertTrue(groupedTopNBuilder.processPage(input.get(i)).process()); assertBuilderSize(groupByHash, types, Collections.nCopies(i + 1, rowCount), Collections.nCopies(rowCount, i + 1), groupedTopNBuilder.getEstimatedSizeInBytes()); Iterator<Page> output = groupedTopNBuilder.buildResult(); while (output.hasNext()) { remainingRows -= output.next().getPositionCount(); .addAll(Collections.nCopies(rowCount - (remainingRows + pageCount - 1) / pageCount, 0)) .build(), groupedTopNBuilder.getEstimatedSizeInBytes()); outputPageCount++; assertBuilderSize(groupByHash, types, Collections.nCopies(pageCount, 0), Collections.nCopies(rowCount, 0), groupedTopNBuilder.getEstimatedSizeInBytes());
.build(); GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), assertTrue(groupedTopNBuilder.processPage(input.get(0)).process()); assertEquals(groupedTopNBuilder.getBufferedPages().size(), 1); Page firstCompactPage = groupedTopNBuilder.getBufferedPages().get(0); Page expected = rowPagesBuilder(types) .row(1L, 0.8) assertTrue(groupedTopNBuilder.processPage(input.get(1)).process()); assertEquals(groupedTopNBuilder.getBufferedPages().size(), 1); assertEquals(firstCompactPage, groupedTopNBuilder.getBufferedPages().get(0)); assertTrue(groupedTopNBuilder.processPage(input.get(2)).process()); List<Page> bufferedPages = groupedTopNBuilder.getBufferedPages(); assertEquals(bufferedPages.size(), 2); assertTrue(groupedTopNBuilder.processPage(input.get(3)).process()); bufferedPages = groupedTopNBuilder.getBufferedPages(); assertEquals(bufferedPages.size(), 2); assertTrue(groupedTopNBuilder.processPage(input.get(4)).process()); bufferedPages = groupedTopNBuilder.getBufferedPages(); assertEquals(bufferedPages.size(), 2);
@Benchmark public List<Page> topN(BenchmarkData data) { data.getTopNBuilder().processPage(data.getPage()).process(); return ImmutableList.copyOf(data.getTopNBuilder().buildResult()); }
public TopNOperator( OperatorContext operatorContext, List<Type> types, int n, List<Integer> sortChannels, List<SortOrder> sortOrders) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); checkArgument(n >= 0, "n must be positive"); if (n == 0) { finishing = true; outputIterator = emptyIterator(); } else { topNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, sortChannels, sortOrders), n, false, new NoChannelGroupByHash()); } }
@Override public Page getOutput() { if (!finishing || noMoreOutput()) { return null; } if (outputIterator == null) { // start flushing outputIterator = topNBuilder.buildResult(); } Page output = null; if (outputIterator.hasNext()) { output = outputIterator.next(); } else { outputIterator = emptyIterator(); } updateMemoryReservation(); return output; }
GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), false, groupByHash); assertBuilderSize(groupByHash, types, ImmutableList.of(), ImmutableList.of(), groupedTopNBuilder.getEstimatedSizeInBytes()); Work<?> work = groupedTopNBuilder.processPage(input); assertFalse(work.process()); assertFalse(work.process()); unblock.set(true); assertTrue(work.process()); List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult()); assertEquals(output.size(), 1); .get(0); assertPageEquals(types, output.get(0), expected); assertBuilderSize(groupByHash, types, ImmutableList.of(0), ImmutableList.of(), groupedTopNBuilder.getEstimatedSizeInBytes());
this.groupedTopNBuilder = new GroupedTopNBuilder( ImmutableList.copyOf(sourceTypes), new SimplePageWithPositionComparator(types, sortChannels, sortOrders),
outputIterator = groupedTopNBuilder.buildResult();
GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), produceRowNumbers, groupByHash); assertBuilderSize(groupByHash, types, ImmutableList.of(), ImmutableList.of(), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(0)).process()); assertBuilderSize(groupByHash, types, ImmutableList.of(4), ImmutableList.of(1, 1, 2), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(1)).process()); assertBuilderSize(groupByHash, types, ImmutableList.of(4, 1), ImmutableList.of(2, 1, 2), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(2)).process()); assertBuilderSize(groupByHash, types, ImmutableList.of(4, 1, 2), ImmutableList.of(2, 2, 2, 1), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(3)).process()); assertBuilderSize(groupByHash, types, ImmutableList.of(4, 1, 2, 0), ImmutableList.of(2, 2, 2, 1), groupedTopNBuilder.getEstimatedSizeInBytes()); List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult()); assertEquals(output.size(), 1); assertBuilderSize(groupByHash, types, ImmutableList.of(0, 0, 0, 0), ImmutableList.of(0, 0, 0, 0), groupedTopNBuilder.getEstimatedSizeInBytes());
@Setup public void setup() { page = createInputPage(Integer.valueOf(positions), types); topNBuilder = new GroupedTopNBuilder(types, comparator, Integer.valueOf(topN), false, new NoChannelGroupByHash()); }
public Work<?> processPage(Page page) { return new TransformWork<>( groupByHash.getGroupIds(page), groupIds -> { processPage(page, groupIds); return null; }); }
private boolean updateMemoryReservation() { // TODO: may need to use trySetMemoryReservation with a compaction to free memory (but that may cause GC pressure) localUserMemoryContext.setBytes(groupedTopNBuilder.getEstimatedSizeInBytes()); return operatorContext.isWaitingForMemory().isDone(); }
GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST)), produceRowNumbers, new NoChannelGroupByHash()); assertBuilderSize(new NoChannelGroupByHash(), types, ImmutableList.of(), ImmutableList.of(), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(0)).process()); assertBuilderSize(new NoChannelGroupByHash(), types, ImmutableList.of(4), ImmutableList.of(4), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(1)).process()); assertBuilderSize(new NoChannelGroupByHash(), types, ImmutableList.of(4, 1), ImmutableList.of(5), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(2)).process()); assertBuilderSize(new NoChannelGroupByHash(), types, ImmutableList.of(4, 1, 1), ImmutableList.of(5), groupedTopNBuilder.getEstimatedSizeInBytes()); assertTrue(groupedTopNBuilder.processPage(input.get(3)).process()); assertBuilderSize(new NoChannelGroupByHash(), types, ImmutableList.of(4, 1, 1), ImmutableList.of(5), groupedTopNBuilder.getEstimatedSizeInBytes()); List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult()); assertEquals(output.size(), 1); assertBuilderSize(new NoChannelGroupByHash(), types, ImmutableList.of(0, 0, 0), ImmutableList.of(0), groupedTopNBuilder.getEstimatedSizeInBytes());
@Override public void addInput(Page page) { checkState(!finishing, "Operator is already finishing"); boolean done = topNBuilder.processPage(requireNonNull(page, "page is null")).process(); // there is no grouping so work will always be done verify(done); updateMemoryReservation(); }