@Override public int getBucket(Page page, int position) { return generator.getPartition(bucketCount, position, page); }
public long getRawHash(Page page, int position) { return hashGenerator.hashPosition(position, page); }
private static PageWithPositionComparator createHashPageWithPositionComparator(HashGenerator hashGenerator) { return (Page leftPage, int leftPosition, Page rightPage, int rightPosition) -> { long leftHash = hashGenerator.hashPosition(leftPosition, leftPage); long rightHash = hashGenerator.hashPosition(rightPosition, rightPage); return Long.compare(leftHash, rightHash); }; }
@Override public int getBucket(Page page, int position) { return generator.getPartition(bucketCount, position, page); }
private int putIfAbsent(int position, Page page) { long rawHash = hashGenerator.hashPosition(position, page); return putIfAbsent(position, page, rawHash); }
public static long getHashPosition(List<? extends Type> hashTypes, Block[] hashBlocks, int position) { int[] hashChannels = new int[hashBlocks.length]; for (int i = 0; i < hashBlocks.length; i++) { hashChannels[i] = i; } HashGenerator hashGenerator = new InterpretedHashGenerator(ImmutableList.copyOf(hashTypes), hashChannels); Page page = new Page(hashBlocks); return hashGenerator.hashPosition(position, page); }
public static Block getHashBlock(List<? extends Type> hashTypes, Block... hashBlocks) { checkArgument(hashTypes.size() == hashBlocks.length); int[] hashChannels = new int[hashBlocks.length]; for (int i = 0; i < hashBlocks.length; i++) { hashChannels[i] = i; } HashGenerator hashGenerator = new InterpretedHashGenerator(ImmutableList.copyOf(hashTypes), hashChannels); int positionCount = hashBlocks[0].getPositionCount(); BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(positionCount); Page page = new Page(hashBlocks); for (int i = 0; i < positionCount; i++) { BIGINT.writeLong(builder, hashGenerator.hashPosition(i, page)); } return builder.build(); }
private int getGroupId(HashGenerator hashGenerator, Page page, int positionInDictionary) { if (dictionaryLookBack.isProcessed(positionInDictionary)) { return dictionaryLookBack.getGroupId(positionInDictionary); } int groupId = putIfAbsent(positionInDictionary, page, hashGenerator.hashPosition(positionInDictionary, page)); dictionaryLookBack.setProcessed(positionInDictionary, groupId); return groupId; }
@Override public long getJoinPosition(int position, Page page) { return getJoinPosition(position, page, hashGenerator.hashPosition(position, page)); }
@Override public int putIfAbsent(int position, Page page) { int rawHash = hashGenerator.hashPosition(position, page); return putIfAbsent(position, page, rawHash); }
public static int getHashPosition(List<? extends Type> hashTypes, Block[] hashBlocks, int position) { int[] hashChannels = new int[hashBlocks.length]; for (int i = 0; i < hashBlocks.length; i++) { hashChannels[i] = i; } HashGenerator hashGenerator = new InterpretedHashGenerator(ImmutableList.copyOf(hashTypes), hashChannels); Page page = new Page(hashBlocks); return hashGenerator.hashPosition(position, page); }
public static Block getHashBlock(List<? extends Type> hashTypes, Block... hashBlocks) { checkArgument(hashTypes.size() == hashBlocks.length); int[] hashChannels = new int[hashBlocks.length]; for (int i = 0; i < hashBlocks.length; i++) { hashChannels[i] = i; } HashGenerator hashGenerator = new InterpretedHashGenerator(ImmutableList.copyOf(hashTypes), hashChannels); int positionCount = hashBlocks[0].getPositionCount(); BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(positionCount); Page page = new Page(hashBlocks); for (int i = 0; i < positionCount; i++) { BIGINT.writeLong(builder, hashGenerator.hashPosition(i, page)); } return builder.build(); }
private int getGroupId(HashGenerator hashGenerator, Page page, int positionInDictionary) { if (dictionaryLookBack.isProcessed(positionInDictionary)) { return dictionaryLookBack.getGroupId(positionInDictionary); } int groupId = putIfAbsent(positionInDictionary, page, hashGenerator.hashPosition(positionInDictionary, page)); dictionaryLookBack.setProcessed(positionInDictionary, groupId); return groupId; }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(!isFinished(), "Operator is already finished"); // build a block containing the partition id of each position BlockBuilder blockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), page.getPositionCount()); for (int position = 0; position < page.getPositionCount(); position++) { int rawHash = hashGenerator.hashPosition(position, page); int partition = murmurHash3(rawHash) & parallelStreamMask; BIGINT.writeLong(blockBuilder, partition); } Block partitionIds = blockBuilder.build(); long size = 0; for (int partition = 0; partition < partitions.length; partition++) { PagesIndex index = partitions[partition]; index.addPage(page, partition, partitionIds); size += index.getEstimatedSize().toBytes(); } operatorContext.setMemoryReservation(size); operatorContext.recordGeneratedOutput(page.getSizeInBytes(), page.getPositionCount()); }
int rawHash = hashGenerator.hashPosition(position, page);