MemorySegment[] close() throws IOException { final MemorySegment current = getCurrentSegment(); if (current == null) { throw new IllegalStateException("Illegal State in HashPartition: No current buffer when finilizing build side."); } finalizeSegment(current, getCurrentPositionInSegment()); clear(); if (this.writer == null) { this.targetList.add(current); MemorySegment[] buffers = this.targetList.toArray(new MemorySegment[this.targetList.size()]); this.targetList.clear(); return buffers; } else { writer.writeBlock(current); return null; } }
return this.buildSideWriteBuffer.spill(this.buildSideChannel);
public void clearAllMemory(List<MemorySegment> target) { if (this.buildSideWriteBuffer.getCurrentSegment() != null) { target.add(this.buildSideWriteBuffer.getCurrentSegment());
return this.buildSideWriteBuffer.spill(this.buildSideChannel);
public void clearAllMemory(List<MemorySegment> target) { if (this.buildSideWriteBuffer.getCurrentSegment() != null) { target.add(this.buildSideWriteBuffer.getCurrentSegment());
public void clearAllMemory(List<MemorySegment> target) { if (this.buildSideWriteBuffer.getCurrentSegment() != null) { target.add(this.buildSideWriteBuffer.getCurrentSegment());
public void clearAllMemory(List<MemorySegment> target) { if (this.buildSideWriteBuffer.getCurrentSegment() != null) { target.add(this.buildSideWriteBuffer.getCurrentSegment());
return this.buildSideWriteBuffer.spill(this.buildSideChannel);
return this.buildSideWriteBuffer.spill(this.buildSideChannel);
/** * Creates a new partition, initially in memory, with one buffer for the build side. The partition is * initialized to expect record insertions for the build side. * * @param partitionNumber The number of the partition. * @param recursionLevel The recursion level - zero for partitions from the initial build, <i>n + 1</i> for * partitions that are created from spilled partition with recursion level <i>n</i>. * @param initialBuffer The initial buffer for this partition. */ HashPartition(TypeSerializer<BT> buildSideAccessors, TypeSerializer<PT> probeSideAccessors, int partitionNumber, int recursionLevel, MemorySegment initialBuffer, MemorySegmentSource memSource, int segmentSize) { super(0); this.buildSideSerializer = buildSideAccessors; this.probeSideSerializer = probeSideAccessors; this.partitionNumber = partitionNumber; this.recursionLevel = recursionLevel; this.memorySegmentSize = segmentSize; this.segmentSizeBits = MathUtils.log2strict(segmentSize); this.overflowSegments = new MemorySegment[2]; this.numOverflowSegments = 0; this.nextOverflowBucket = 0; this.buildSideWriteBuffer = new BuildSideBuffer(initialBuffer, memSource); }
/** * Creates a new partition, initially in memory, with one buffer for the build side. The partition is * initialized to expect record insertions for the build side. * * @param partitionNumber The number of the partition. * @param recursionLevel The recursion level - zero for partitions from the initial build, <i>n + 1</i> for * partitions that are created from spilled partition with recursion level <i>n</i>. * @param initialBuffer The initial buffer for this partition. */ HashPartition(TypeSerializer<BT> buildSideAccessors, TypeSerializer<PT> probeSideAccessors, int partitionNumber, int recursionLevel, MemorySegment initialBuffer, MemorySegmentSource memSource, int segmentSize) { super(0); this.buildSideSerializer = buildSideAccessors; this.probeSideSerializer = probeSideAccessors; this.partitionNumber = partitionNumber; this.recursionLevel = recursionLevel; this.memorySegmentSize = segmentSize; this.segmentSizeBits = MathUtils.log2strict(segmentSize); this.overflowSegments = new MemorySegment[2]; this.numOverflowSegments = 0; this.nextOverflowBucket = 0; this.buildSideWriteBuffer = new BuildSideBuffer(initialBuffer, memSource); }
/** * Creates a new partition, initially in memory, with one buffer for the build side. The partition is * initialized to expect record insertions for the build side. * * @param partitionNumber The number of the partition. * @param recursionLevel The recursion level - zero for partitions from the initial build, <i>n + 1</i> for * partitions that are created from spilled partition with recursion level <i>n</i>. * @param initialBuffer The initial buffer for this partition. */ HashPartition(TypeSerializer<BT> buildSideAccessors, TypeSerializer<PT> probeSideAccessors, int partitionNumber, int recursionLevel, MemorySegment initialBuffer, MemorySegmentSource memSource, int segmentSize) { super(0); this.buildSideSerializer = buildSideAccessors; this.probeSideSerializer = probeSideAccessors; this.partitionNumber = partitionNumber; this.recursionLevel = recursionLevel; this.memorySegmentSize = segmentSize; this.segmentSizeBits = MathUtils.log2strict(segmentSize); this.overflowSegments = new MemorySegment[2]; this.numOverflowSegments = 0; this.nextOverflowBucket = 0; this.buildSideWriteBuffer = new BuildSideBuffer(initialBuffer, memSource); }
/** * Creates a new partition, initially in memory, with one buffer for the build side. The partition is * initialized to expect record insertions for the build side. * * @param partitionNumber The number of the partition. * @param recursionLevel The recursion level - zero for partitions from the initial build, <i>n + 1</i> for * partitions that are created from spilled partition with recursion level <i>n</i>. * @param initialBuffer The initial buffer for this partition. */ HashPartition(TypeSerializer<BT> buildSideAccessors, TypeSerializer<PT> probeSideAccessors, int partitionNumber, int recursionLevel, MemorySegment initialBuffer, MemorySegmentSource memSource, int segmentSize) { super(0); this.buildSideSerializer = buildSideAccessors; this.probeSideSerializer = probeSideAccessors; this.partitionNumber = partitionNumber; this.recursionLevel = recursionLevel; this.memorySegmentSize = segmentSize; this.segmentSizeBits = MathUtils.log2strict(segmentSize); this.overflowSegments = new MemorySegment[2]; this.numOverflowSegments = 0; this.nextOverflowBucket = 0; this.buildSideWriteBuffer = new BuildSideBuffer(initialBuffer, memSource); }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
@Override protected MemorySegment nextSegment(MemorySegment current, int bytesUsed) throws IOException { finalizeSegment(current, bytesUsed); final MemorySegment next; if (this.writer == null) { this.targetList.add(current); next = this.memSource.nextSegment(); } else { this.writer.writeBlock(current); try { next = this.writer.getReturnQueue().take(); } catch (InterruptedException iex) { throw new IOException("Hash Join Partition was interrupted while grabbing a new write-behind buffer."); } } this.currentBlockNumber++; return next; }
@Override protected MemorySegment nextSegment(MemorySegment current, int bytesUsed) throws IOException { finalizeSegment(current, bytesUsed); final MemorySegment next; if (this.writer == null) { this.targetList.add(current); next = this.memSource.nextSegment(); } else { this.writer.writeBlock(current); try { next = this.writer.getReturnQueue().take(); } catch (InterruptedException iex) { throw new IOException("Hash Join Partition was interrupted while grabbing a new write-behind buffer."); } } this.currentBlockNumber++; return next; }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }