public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset) { long pageAddress = addresses.getLong(toIntExact(position)); int blockIndex = decodeSliceIndex(pageAddress); int blockPosition = decodePosition(pageAddress); pagesHashStrategy.appendTo(blockIndex, blockPosition, pageBuilder, outputChannelOffset); }
private long hashPosition(long sliceAddress) { int sliceIndex = decodeSliceIndex(sliceAddress); int position = decodePosition(sliceAddress); if (precomputedHashChannel.isPresent()) { return getRawHash(sliceIndex, position); } return hashStrategy.hashPosition(sliceIndex, position); }
public boolean positionEqualsPosition(PagesHashStrategy partitionHashStrategy, int leftPosition, int rightPosition) { long leftAddress = valueAddresses.getLong(leftPosition); int leftPageIndex = decodeSliceIndex(leftAddress); int leftPagePosition = decodePosition(leftAddress); long rightAddress = valueAddresses.getLong(rightPosition); int rightPageIndex = decodeSliceIndex(rightAddress); int rightPagePosition = decodePosition(rightAddress); return partitionHashStrategy.positionEqualsPosition(leftPageIndex, leftPagePosition, rightPageIndex, rightPagePosition); }
assertEquals(hashStrategy.getChannelCount(), 1); assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashPosition(VARCHAR, leftBlock, leftBlockPosition)); assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition)); for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected); assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected); assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected); assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected); for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected); assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected); assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected); assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected); hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
assertEquals(hashStrategy.getChannelCount(), 1); assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashPosition(VARCHAR, leftBlock, leftBlockPosition)); assertTrue(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition)); for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, rightBlock), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Block[] {leftBlock}, rightBlockPosition, new Block[] {rightBlock}), expected); assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected); for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, rightBlock), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Block[] {leftBlock}, rightBlockPosition, new Block[] {rightBlock}), expected); assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected); hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
this.channelCount = pagesHashStrategy.getChannelCount(); size = sizeOf(addresses.elements()) + pagesHashStrategy.getSizeInBytes() + sizeOf(key) + sizeOf(positionToHashes); hashCollisions = hashCollisionsLocal;
private int findNextGroupStart(int startPosition, Page page) { for (int i = startPosition + 1; i < page.getPositionCount(); i++) { if (!pagesHashStrategy.rowEqualsRow(startPosition, page, i, page)) { return i; } } return page.getPositionCount(); }
public boolean positionEqualsRow(PagesHashStrategy pagesHashStrategy, int indexPosition, int rightPosition, Page rightPage) { long pageAddress = valueAddresses.getLong(indexPosition); int pageIndex = decodeSliceIndex(pageAddress); int pagePosition = decodePosition(pageAddress); return pagesHashStrategy.positionEqualsRow(pageIndex, pagePosition, rightPosition, rightPage); }
@Override public boolean contains(int position, Page page, int[] hashChannels) { int rawHash = hashStrategy.hashRow(position, page.getBlocks()); int hashPosition = getHashPosition(rawHash, mask); // look for a slot containing this key while (groupAddressByHash[hashPosition] != -1) { long address = groupAddressByHash[hashPosition]; if (hashStrategy.positionEqualsRow(decodeSliceIndex(address), decodePosition(address), position, page, hashChannels)) { // found an existing slot for this key return true; } // increment position and mask to handle wrap around hashPosition = (hashPosition + 1) & mask; } return false; }
public int getAddressIndex(int position, Page hashChannelsPage) { return getAddressIndex(position, hashChannelsPage, pagesHashStrategy.hashRow(position, hashChannelsPage)); }
private boolean positionEqualsCurrentRowIgnoreNulls(int leftPosition, byte rawHash, int rightPosition, Page rightPage) { if (positionToHashes[leftPosition] != rawHash) { return false; } long pageAddress = addresses.getLong(leftPosition); int blockIndex = decodeSliceIndex(pageAddress); int blockPosition = decodePosition(pageAddress); return pagesHashStrategy.positionEqualsRowIgnoreNulls(blockIndex, blockPosition, rightPosition, rightPage); }
private boolean positionEqualsPositionIgnoreNulls(int leftPosition, int rightPosition) { long leftPageAddress = addresses.getLong(leftPosition); int leftBlockIndex = decodeSliceIndex(leftPageAddress); int leftBlockPosition = decodePosition(leftPageAddress); long rightPageAddress = addresses.getLong(rightPosition); int rightBlockIndex = decodeSliceIndex(rightPageAddress); int rightBlockPosition = decodePosition(rightPageAddress); return pagesHashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition); }
assertEquals(hashStrategy.getChannelCount(), outputChannels.size()); .mapToLong(Block::getRetainedSizeInBytes) .sum(); assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes); hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition)); assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition)); assertTrue(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition)); for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { assertEquals( hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition)); assertEquals( hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition)); boolean expected = expectedHashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)), expected); assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected); hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
assertEquals(hashStrategy.getChannelCount(), types.size()); .mapToLong(Block::getRetainedSizeInBytes) .sum(); assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes); hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition)); assertTrue(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition)); for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { assertEquals( hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition)); boolean expected = expectedHashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, rightBlocks); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, rightBlocks), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, leftBlocks, rightPosition, rightBlocks), expected); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, rightBlocks), expected); hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
this.channelCount = pagesHashStrategy.getChannelCount(); + sizeOf(addresses.elements()) + pagesHashStrategy.getSizeInBytes();
private static int findGroupEnd(Page page, PagesHashStrategy pagesHashStrategy, int startPosition) { checkArgument(page.getPositionCount() > 0, "Must have at least one position"); checkPositionIndex(startPosition, page.getPositionCount(), "startPosition out of bounds"); return findEndPosition(startPosition, page.getPositionCount(), (firstPosition, secondPosition) -> pagesHashStrategy.rowEqualsRow(firstPosition, page, secondPosition, page)); }
private boolean positionEqualsCurrentRow(int sliceIndex, int slicePosition, int position, Page page) { return hashStrategy.positionEqualsRow(sliceIndex, slicePosition, position, page, channels); }
@Override public boolean contains(int position, Page page, int[] hashChannels) { long rawHash = hashStrategy.hashRow(position, page); int hashPosition = (int) getHashPosition(rawHash, mask); // look for a slot containing this key while (groupAddressByHash[hashPosition] != -1) { if (positionNotDistinctFromCurrentRow(groupAddressByHash[hashPosition], hashPosition, position, page, (byte) rawHash, hashChannels)) { // found an existing slot for this key return true; } // increment position and mask to handle wrap around hashPosition = (hashPosition + 1) & mask; } return false; }
private long readHashPosition(int position) { long pageAddress = addresses.getLong(position); int blockIndex = decodeSliceIndex(pageAddress); int blockPosition = decodePosition(pageAddress); return pagesHashStrategy.hashPosition(blockIndex, blockPosition); }
private void processInput(Page page) { requireNonNull(page, "page is null"); Page groupByPage = extractColumns(page, groupByChannels); if (currentGroup != null) { if (!pagesHashStrategy.rowEqualsRow(0, extractColumns(currentGroup, groupByChannels), 0, groupByPage)) { // page starts with new group, so flush it evaluateAndFlushGroup(currentGroup, 0); } currentGroup = null; } int startPosition = 0; while (true) { // may be equal to page.getPositionCount() if the end is not found in this page int nextGroupStart = findNextGroupStart(startPosition, groupByPage); addRowsToAggregates(page, startPosition, nextGroupStart - 1); if (nextGroupStart < page.getPositionCount()) { // current group stops somewhere in the middle of the page, so flush it evaluateAndFlushGroup(page, startPosition); startPosition = nextGroupStart; } else { currentGroup = page.getRegion(page.getPositionCount() - 1, 1); return; } } }