@Override public LookupSource getLookupSource() { return new PartitionedLookupSource( partitions.stream() .map(Supplier::get) .collect(toImmutableList()), hashChannelTypes, Optional.of(outerPositionTrackerFactory.create())); }
@Override public long getNextJoinPosition(long currentJoinPosition, int probePosition, Page allProbeChannelsPage) { int partition = decodePartition(currentJoinPosition); long joinPosition = decodeJoinPosition(currentJoinPosition); LookupSource lookupSource = lookupSources[partition]; long nextJoinPosition = lookupSource.getNextJoinPosition(joinPosition, probePosition, allProbeChannelsPage); if (nextJoinPosition < 0) { return nextJoinPosition; } return encodePartitionedJoinPosition(partition, toIntExact(nextJoinPosition)); }
@Override public long getJoinPosition(int position, Page hashChannelsPage, Page allChannelsPage) { return getJoinPosition(position, hashChannelsPage, allChannelsPage, partitionGenerator.getRawHash(hashChannelsPage, position)); }
@Override public boolean isJoinPositionEligible(long currentJoinPosition, int probePosition, Page allProbeChannelsPage) { int partition = decodePartition(currentJoinPosition); long joinPosition = decodeJoinPosition(currentJoinPosition); LookupSource lookupSource = lookupSources[partition]; return lookupSource.isJoinPositionEligible(joinPosition, probePosition, allProbeChannelsPage); }
@Override public long getJoinPosition(int position, Page hashChannelsPage, Page allChannelsPage, long rawHash) { int partition = partitionGenerator.getPartition(rawHash); LookupSource lookupSource = lookupSources[partition]; long joinPosition = lookupSource.getJoinPosition(position, hashChannelsPage, allChannelsPage, rawHash); if (joinPosition < 0) { return joinPosition; } return encodePartitionedJoinPosition(partition, toIntExact(joinPosition)); }
this.lookupSourceSupplier = createPartitionedLookupSourceSupplier(partitions, hashChannelTypes, outer);
@Override public long joinPositionWithinPartition(long joinPosition) { return decodeJoinPosition(joinPosition); }
@Override public long getNextJoinPosition(long partitionedJoinPosition) { int partition = (int) (partitionedJoinPosition & partitionMask); long joinPosition = partitionedJoinPosition >>> lookupSources.length; LookupSource lookupSource = lookupSources[partition]; long nextJoinPosition = lookupSource.getNextJoinPosition(joinPosition); if (nextJoinPosition < 0) { return nextJoinPosition; } return encodePartitionedJoinPosition(partition, nextJoinPosition); }
@Override public void appendTo(long partitionedJoinPosition, PageBuilder pageBuilder, int outputChannelOffset) { int partition = decodePartition(partitionedJoinPosition); int joinPosition = decodeJoinPosition(partitionedJoinPosition); lookupSources[partition].appendTo(joinPosition, pageBuilder, outputChannelOffset); if (outerPositionTracker != null) { outerPositionTracker.positionVisited(partition, joinPosition); } }
verify(!outer, "It is not possible to reset lookupSourceSupplier which is tracking for outer join"); verify(partitions.length > 1, "Spill occurred when only one partition"); lookupSourceSupplier = createPartitionedLookupSourceSupplier(ImmutableList.copyOf(partitions), hashChannelTypes, outer); closeCachedLookupSources();
() -> new PartitionedLookupSource( partitions.stream() .map(Supplier::get)
@Override public long getJoinPosition(int position, Page page, int rawHash) { int partition = murmurHash3(rawHash) & partitionMask; LookupSource lookupSource = lookupSources[partition]; long joinPosition = lookupSource.getJoinPosition(position, page, rawHash); if (joinPosition < 0) { return joinPosition; } return encodePartitionedJoinPosition(partition, joinPosition); }
@Override public long getJoinPosition(int position, Page page) { return getJoinPosition(position, page, hashGenerator.hashPosition(position, page)); }
public ParallelLookupSourceSupplier(List<Type> types, List<Integer> hashChannels, List<? extends ListenableFuture<SharedLookupSource>> partitions) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); this.partitions = requireNonNull(partitions, "partitions is null"); hashChannelTypes = hashChannels.stream() .map(types::get) .collect(toImmutableList()); checkArgument(Integer.bitCount(partitions.size()) == 1, "partitions must be a power of 2"); lookupSourceFuture = Futures.transform(Futures.allAsList(partitions), (List<SharedLookupSource> input) -> { return new PartitionedLookupSource(input, hashChannelTypes); }); }