@Override public void close() { if (closed) { return; } closed = true; pageBuilder.reset(); onClose.run(); } }
public void reset() { // be aware that probeIndexBuilder will not clear its capacity probeIndexBuilder.clear(); buildPageBuilder.reset(); estimatedProbeBlockBytes = 0; isSequentialProbeIndices = true; }
private void flushPage() { if (!pageBuilder.isEmpty()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } }
private void flush() { if (!pageBuilder.isEmpty()) { Page output = pageBuilder.build(); pageBuilder.reset(); outputQueue.add(output); } }
@Override public void finish() { finishing = true; if (currentGroup != null) { evaluateAndFlushGroup(currentGroup, 0); currentGroup = null; } if (!pageBuilder.isEmpty()) { outputPages.add(pageBuilder.build()); pageBuilder.reset(); } }
private synchronized ListenableFuture<?> flush(int partition) { PageBuilder pageBuilder = pageBuilders[partition]; if (pageBuilder.isEmpty()) { return Futures.immediateFuture(null); } Page page = pageBuilder.build(); pageBuilder.reset(); return getSpiller(partition).spill(page); }
@Override public Page getOutput() { // only return a page if the page buffer isFull or we are finishing and the page buffer has data if (pageBuilder.isFull() || (finishing && !pageBuilder.isEmpty())) { Page page = pageBuilder.build(); pageBuilder.reset(); return page; } return null; }
@Override protected Page computeNext() { while (!pageBuilder.isFull() && pagePositions.hasNext()) { pagePositions.next().appendTo(pageBuilder); } if (pageBuilder.isEmpty()) { return endOfData(); } Page page = pageBuilder.build(); pageBuilder.reset(); return page; }
public void flush(boolean force) { // add all full pages to output buffer for (int partition = 0; partition < pageBuilders.length; partition++) { PageBuilder partitionPageBuilder = pageBuilders[partition]; if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) { Page pagePartition = partitionPageBuilder.build(); partitionPageBuilder.reset(); List<SerializedPage> serializedPages = splitPage(pagePartition, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream() .map(serde::serialize) .collect(toImmutableList()); outputBuffer.enqueue(partition, serializedPages); pagesAdded.incrementAndGet(); rowsAdded.addAndGet(pagePartition.getPositionCount()); } } } }
private void evaluateAndFlushGroup(Page page, int position) { pageBuilder.declarePosition(); for (int i = 0; i < groupByTypes.size(); i++) { Block block = page.getBlock(groupByChannels[i]); Type type = groupByTypes.get(i); type.appendTo(block, position, pageBuilder.getBlockBuilder(i)); } int offset = groupByTypes.size(); for (int i = 0; i < aggregates.size(); i++) { aggregates.get(i).evaluate(pageBuilder.getBlockBuilder(offset + i)); } if (pageBuilder.isFull()) { outputPages.add(pageBuilder.build()); pageBuilder.reset(); } aggregates = setupAggregates(step, accumulatorFactories); }
@Override public Page computeNext() { currentPosition = buildPage(currentPosition, outputChannels, pageBuilder); if (pageBuilder.isEmpty()) { return endOfData(); } Page page = pageBuilder.build(); pageBuilder.reset(); return page; } };
private Block computeResultBlock(Type type, Block block, int arrayLength) { if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); for (int i = 0; i < arrayLength; ++i) { type.appendTo(block, positions.get(i), blockBuilder); } pageBuilder.declarePositions(arrayLength); return blockBuilder.getRegion(blockBuilder.getPositionCount() - arrayLength, arrayLength); }
@Override public Page getOutput() { if (state != State.HAS_OUTPUT) { return null; } if (currentPosition >= pageIndex.getPositionCount()) { state = State.FINISHED; return null; } // iterate through the positions sequentially until we have one full page pageBuilder.reset(); currentPosition = pageIndex.buildPage(currentPosition, outputChannels, pageBuilder); // output the page if we have any data if (pageBuilder.isEmpty()) { state = State.FINISHED; return null; } Page page = pageBuilder.build(); return page; }
private static List<Page> createOutputPages(List<Type> types, List<Page> inputPages, long[] sortedAddresses) { PageBuilder pageBuilder = new PageBuilder(types); pageBuilder.reset(); for (long address : sortedAddresses) { int index = sorter.decodePageIndex(address); int position = sorter.decodePositionIndex(address); Page page = inputPages.get(index); for (int i = 0; i < types.size(); i++) { Type type = types.get(i); type.appendTo(page.getBlock(i), position, pageBuilder.getBlockBuilder(i)); } pageBuilder.declarePosition(); } return ImmutableList.of(pageBuilder.build()); } }
private static Block featuresHelper(PageBuilder pageBuilder, double... features) { if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder mapBlockBuilder = pageBuilder.getBlockBuilder(0); BlockBuilder blockBuilder = mapBlockBuilder.beginBlockEntry(); for (int i = 0; i < features.length; i++) { BigintType.BIGINT.writeLong(blockBuilder, i); DoubleType.DOUBLE.writeDouble(blockBuilder, features[i]); } mapBlockBuilder.closeEntry(); pageBuilder.declarePosition(); return mapBlockBuilder.getObject(mapBlockBuilder.getPositionCount() - 1, Block.class); } }
@TypeParameter("E") @SqlType("array(E)") public Block reverse( @TypeParameter("E") Type type, @SqlType("array(E)") Block block) { int arrayLength = block.getPositionCount(); if (arrayLength < 2) { return block; } if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); for (int i = arrayLength - 1; i >= 0; i--) { type.appendTo(block, i, blockBuilder); } pageBuilder.declarePositions(arrayLength); return blockBuilder.getRegion(blockBuilder.getPositionCount() - arrayLength, arrayLength); } }
@SqlType("row(x integer,y integer)") public Block bingTileCoordinates(@SqlType(BingTileType.NAME) long input) { if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); BingTile tile = BingTile.decode(input); BlockBuilder tileBlockBuilder = blockBuilder.beginBlockEntry(); INTEGER.writeLong(tileBlockBuilder, tile.getX()); INTEGER.writeLong(tileBlockBuilder, tile.getY()); blockBuilder.closeEntry(); pageBuilder.declarePosition(); return BING_TILE_COORDINATES_ROW_TYPE.getObject(blockBuilder, blockBuilder.getPositionCount() - 1); } }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object bigintGroupByHash(SingleChannelBenchmarkData data) { GroupByHash groupByHash = new BigintGroupByHash(0, data.getHashEnabled(), EXPECTED_SIZE, NOOP); data.getPages().forEach(p -> groupByHash.addPage(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object groupByHashPreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP); data.getPages().forEach(p -> groupByHash.getGroupIds(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object addPagePreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP); data.getPages().forEach(p -> groupByHash.addPage(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }