@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, WindowOperator.class.getSimpleName()); return new WindowOperator( operatorContext, sourceTypes, outputChannels, windowFunctionDefinitions, partitionChannels, preGroupedChannels, sortChannels, sortOrder, preSortedChannelPrefix, expectedPositions, pagesIndexFactory); }
@Override public Page getOutput() { if (state == State.NEEDS_INPUT || state == State.FINISHED) { return null; } Page page = extractOutput(); localUserMemoryContext.setBytes(pagesIndex.getEstimatedSize().toBytes()); return page; }
private static void assertFindEndPosition(String values, int expected) { char[] array = values.toCharArray(); assertEquals(WindowOperator.findEndPosition(0, array.length, (first, second) -> array[first] == array[second]), expected); }
/** * @return true if a full group has been buffered after processing the pendingInput, false otherwise */ private boolean processPendingInput() { checkState(pendingInput != null); pendingInput = updatePagesIndex(pendingInput); // If we have unused input or are finishing, then we have buffered a full group if (pendingInput != null || state == State.FINISHING) { finishPagesIndex(); return true; } else { return false; } }
/** * @return the unused section of the page, or null if fully applied. * pagesIndex guaranteed to have at least one row after this method returns */ private Page updatePagesIndex(Page page) { checkArgument(page.getPositionCount() > 0); // TODO: Fix pagesHashStrategy to allow specifying channels for comparison, it currently requires us to rearrange the right side blocks in consecutive channel order Page preGroupedPage = rearrangePage(page, preGroupedChannels); if (pagesIndex.getPositionCount() == 0 || pagesIndex.positionEqualsRow(preGroupedPartitionHashStrategy, 0, 0, preGroupedPage)) { // Find the position where the pre-grouped columns change int groupEnd = findGroupEnd(preGroupedPage, preGroupedPartitionHashStrategy, 0); // Add the section of the page that contains values for the current group pagesIndex.addPage(page.getRegion(0, groupEnd)); if (page.getPositionCount() - groupEnd > 0) { // Save the remaining page, which may contain multiple partitions return page.getRegion(groupEnd, page.getPositionCount() - groupEnd); } else { // Page fully consumed return null; } } else { // We had previous results buffered, but the new page starts with new group values return page; } }
if (pendingInput != null && processPendingInput()) { partitionStart = 0; int partitionEnd = findGroupEnd(pagesIndex, unGroupedPartitionHashStrategy, partitionStart); partition = new WindowPartition(pagesIndex, partitionStart, partitionEnd, outputChannels, windowFunctions, peerGroupHashStrategy); windowInfo.addPartition(partition);
/** * @return true if a full group has been buffered after processing the pendingInput, false otherwise */ private boolean processPendingInput() { checkState(pendingInput != null); pendingInput = updatePagesIndex(pendingInput); // If we have unused input or are finishing, then we have buffered a full group if (pendingInput != null || state == State.FINISHING) { sortPagesIndexIfNecessary(); return true; } else { return false; } }
private void finishPagesIndex() { sortPagesIndexIfNecessary(); windowInfo.addIndex(pagesIndex); }
private void sortPagesIndexIfNecessary() { if (pagesIndex.getPositionCount() > 1 && !orderChannels.isEmpty()) { int startPosition = 0; while (startPosition < pagesIndex.getPositionCount()) { int endPosition = findGroupEnd(pagesIndex, preSortedPartitionHashStrategy, startPosition); pagesIndex.sort(orderChannels, ordering, startPosition, endPosition); startPosition = endPosition; } } }
@Override public void addInput(Page page) { checkState(state == State.NEEDS_INPUT, "Operator can not take input at this time"); requireNonNull(page, "page is null"); checkState(pendingInput == null, "Operator already has pending input"); if (page.getPositionCount() == 0) { return; } pendingInput = page; if (processPendingInput()) { state = State.HAS_OUTPUT; } localUserMemoryContext.setBytes(pagesIndex.getEstimatedSize().toBytes()); }
@Override public void finish() { if (state == State.FINISHING || state == State.FINISHED) { return; } if (state == State.NEEDS_INPUT) { // Since was waiting for more input, prepare what we have for output since we will not be getting any more input finishPagesIndex(); } state = State.FINISHING; }
/** * @return the unused section of the page, or null if fully applied. * pagesIndex guaranteed to have at least one row after this method returns */ private Page updatePagesIndex(Page page) { checkArgument(page.getPositionCount() > 0); // TODO: Fix pagesHashStrategy to allow specifying channels for comparison, it currently requires us to rearrange the right side blocks in consecutive channel order Page preGroupedPage = rearrangePage(page, preGroupedChannels); if (pagesIndex.getPositionCount() == 0 || pagesIndex.positionEqualsRow(preGroupedPartitionHashStrategy, 0, 0, preGroupedPage.getBlocks())) { // Find the position where the pre-grouped columns change int groupEnd = findGroupEnd(preGroupedPage, preGroupedPartitionHashStrategy, 0); // Add the section of the page that contains values for the current group pagesIndex.addPage(page.getRegion(0, groupEnd)); if (page.getPositionCount() - groupEnd > 0) { // Save the remaining page, which may contain multiple partitions return page.getRegion(groupEnd, page.getPositionCount() - groupEnd); } else { // Page fully consumed return null; } } else { // We had previous results buffered, but the new page starts with new group values return page; } }
if (pendingInput != null && processPendingInput()) { partitionStart = 0; int partitionEnd = findGroupEnd(pagesIndex, unGroupedPartitionHashStrategy, partitionStart); partition = new WindowPartition(pagesIndex, partitionStart, partitionEnd, outputChannels, windowFunctions, frameInfo, peerGroupHashStrategy);
@Override public void finish() { if (state == State.FINISHING || state == State.FINISHED) { return; } if (state == State.NEEDS_INPUT) { // Since was waiting for more input, prepare what we have for output since we will not be getting any more input sortPagesIndexIfNecessary(); } state = State.FINISHING; }
private void sortPagesIndexIfNecessary() { if (pagesIndex.getPositionCount() > 1 && !orderChannels.isEmpty()) { int startPosition = 0; while (startPosition < pagesIndex.getPositionCount()) { int endPosition = findGroupEnd(pagesIndex, preSortedPartitionHashStrategy, startPosition); pagesIndex.sort(orderChannels, ordering, startPosition, endPosition); startPosition = endPosition; } } }
@Override public void addInput(Page page) { checkState(state == State.NEEDS_INPUT, "Operator can not take input at this time"); requireNonNull(page, "page is null"); checkState(pendingInput == null, "Operator already has pending input"); if (page.getPositionCount() == 0) { return; } pendingInput = page; if (processPendingInput()) { state = State.HAS_OUTPUT; } operatorContext.setMemoryReservation(pagesIndex.getEstimatedSize().toBytes()); }
private static int findGroupEnd(PagesIndex pagesIndex, PagesHashStrategy pagesHashStrategy, int startPosition) { checkArgument(pagesIndex.getPositionCount() > 0, "Must have at least one position"); checkPositionIndex(startPosition, pagesIndex.getPositionCount(), "startPosition out of bounds"); return findEndPosition(startPosition, pagesIndex.getPositionCount(), (firstPosition, secondPosition) -> pagesIndex.positionEqualsPosition(pagesHashStrategy, firstPosition, secondPosition)); }
@Override public Page getOutput() { if (state == State.NEEDS_INPUT || state == State.FINISHED) { return null; } Page page = extractOutput(); operatorContext.setMemoryReservation(pagesIndex.getEstimatedSize().toBytes()); return page; }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, WindowOperator.class.getSimpleName()); return new WindowOperator( operatorContext, sourceTypes, outputChannels, windowFunctionDefinitions, partitionChannels, preGroupedChannels, sortChannels, sortOrder, preSortedChannelPrefix, frameInfo, expectedPositions); }
private static int findGroupEnd(Page page, PagesHashStrategy pagesHashStrategy, int startPosition) { checkArgument(page.getPositionCount() > 0, "Must have at least one position"); checkPositionIndex(startPosition, page.getPositionCount(), "startPosition out of bounds"); return findEndPosition(startPosition, page.getPositionCount(), (firstPosition, secondPosition) -> pagesHashStrategy.rowEqualsRow(firstPosition, page, secondPosition, page)); }