public ReOpenableMutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer, TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator, TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager, boolean useBitmapFilters) { super(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator, memorySegments, ioManager, useBitmapFilters); keepBuildSidePartitions = true; spilledInMemoryPartitions = ioManager.createChannelEnumerator(); }
public ReOpenableMutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer, TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator, TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager, boolean useBitmapFilters) { super(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator, memorySegments, ioManager, useBitmapFilters); keepBuildSidePartitions = true; spilledInMemoryPartitions = ioManager.createChannelEnumerator(); }
public ReOpenableMutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer, TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator, TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager, boolean useBitmapFilters) { super(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator, memorySegments, ioManager, useBitmapFilters); keepBuildSidePartitions = true; spilledInMemoryPartitions = ioManager.createChannelEnumerator(); }
public ReOpenableMutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer, TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator, TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager, boolean useBitmapFilters) { super(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator, memorySegments, ioManager, useBitmapFilters); keepBuildSidePartitions = true; spilledInMemoryPartitions = ioManager.createChannelEnumerator(); }
protected void createPartitions(int numPartitions, int recursionLevel) { // sanity check ensureNumBuffersReturned(numPartitions); this.currentEnumerator = this.ioManager.createChannelEnumerator(); this.partitionsBeingBuilt.clear(); for (int i = 0; i < numPartitions; i++) { HashPartition<BT, PT> p = getNewInMemoryPartition(i, recursionLevel); this.partitionsBeingBuilt.add(p); } }
protected void createPartitions(int numPartitions, int recursionLevel) { // sanity check ensureNumBuffersReturned(numPartitions); this.currentEnumerator = this.ioManager.createChannelEnumerator(); this.partitionsBeingBuilt.clear(); for (int i = 0; i < numPartitions; i++) { HashPartition<BT, PT> p = getNewInMemoryPartition(i, recursionLevel); this.partitionsBeingBuilt.add(p); } }
public SerializedUpdateBuffer(List<MemorySegment> memSegments, int segmentSize, IOManager ioManager) { super(memSegments.remove(memSegments.size() - 1), segmentSize, HEADER_LENGTH); totalNumBuffers = memSegments.size() + 1; if (totalNumBuffers < 3) { throw new IllegalArgumentException("SerializedUpdateBuffer needs at least 3 memory segments."); } emptyBuffers = new LinkedBlockingQueue<MemorySegment>(totalNumBuffers); fullBuffers = new ArrayDeque<MemorySegment>(64); emptyBuffers.addAll(memSegments); int threshold = (int) ((1 - SPILL_THRESHOLD) * totalNumBuffers); numSegmentsSpillingThreshold = threshold > 0 ? threshold : 0; minBuffersForWriteEnd = Math.max(2, Math.min(16, totalNumBuffers / 2)); minBuffersForSpilledReadEnd = Math.max(1, Math.min(16, totalNumBuffers / 4)); if (minBuffersForSpilledReadEnd + minBuffersForWriteEnd > totalNumBuffers) { throw new IllegalArgumentException("BUG: Unfulfillable memory assignment."); } this.ioManager = ioManager; channelEnumerator = ioManager.createChannelEnumerator(); readEnds = new ArrayList<ReadEnd>(); }
protected void createPartitions(int numPartitions, int recursionLevel) { // sanity check ensureNumBuffersReturned(numPartitions); this.currentEnumerator = this.ioManager.createChannelEnumerator(); this.partitionsBeingBuilt.clear(); for (int i = 0; i < numPartitions; i++) { HashPartition<BT, PT> p = getNewInMemoryPartition(i, recursionLevel); this.partitionsBeingBuilt.add(p); } }
public SerializedUpdateBuffer(List<MemorySegment> memSegments, int segmentSize, IOManager ioManager) { super(memSegments.remove(memSegments.size() - 1), segmentSize, HEADER_LENGTH); totalNumBuffers = memSegments.size() + 1; if (totalNumBuffers < 3) { throw new IllegalArgumentException("SerializedUpdateBuffer needs at least 3 memory segments."); } emptyBuffers = new LinkedBlockingQueue<MemorySegment>(totalNumBuffers); fullBuffers = new ArrayDeque<MemorySegment>(64); emptyBuffers.addAll(memSegments); int threshold = (int) ((1 - SPILL_THRESHOLD) * totalNumBuffers); numSegmentsSpillingThreshold = threshold > 0 ? threshold : 0; minBuffersForWriteEnd = Math.max(2, Math.min(16, totalNumBuffers / 2)); minBuffersForSpilledReadEnd = Math.max(1, Math.min(16, totalNumBuffers / 4)); if (minBuffersForSpilledReadEnd + minBuffersForWriteEnd > totalNumBuffers) { throw new IllegalArgumentException("BUG: Unfulfillable memory assignment."); } this.ioManager = ioManager; channelEnumerator = ioManager.createChannelEnumerator(); readEnds = new ArrayList<ReadEnd>(); }
protected void createPartitions(int numPartitions, int recursionLevel) { // sanity check ensureNumBuffersReturned(numPartitions); this.currentEnumerator = this.ioManager.createChannelEnumerator(); this.partitionsBeingBuilt.clear(); for (int i = 0; i < numPartitions; i++) { HashPartition<BT, PT> p = getNewInMemoryPartition(i, recursionLevel); this.partitionsBeingBuilt.add(p); } }
public SerializedUpdateBuffer(List<MemorySegment> memSegments, int segmentSize, IOManager ioManager) { super(memSegments.remove(memSegments.size() - 1), segmentSize, HEADER_LENGTH); totalNumBuffers = memSegments.size() + 1; if (totalNumBuffers < 3) { throw new IllegalArgumentException("SerializedUpdateBuffer needs at least 3 memory segments."); } emptyBuffers = new LinkedBlockingQueue<MemorySegment>(totalNumBuffers); fullBuffers = new ArrayDeque<MemorySegment>(64); emptyBuffers.addAll(memSegments); int threshold = (int) ((1 - SPILL_THRESHOLD) * totalNumBuffers); numSegmentsSpillingThreshold = threshold > 0 ? threshold : 0; minBuffersForWriteEnd = Math.max(2, Math.min(16, totalNumBuffers / 2)); minBuffersForSpilledReadEnd = Math.max(1, Math.min(16, totalNumBuffers / 4)); if (minBuffersForSpilledReadEnd + minBuffersForWriteEnd > totalNumBuffers) { throw new IllegalArgumentException("BUG: Unfulfillable memory assignment."); } this.ioManager = ioManager; channelEnumerator = ioManager.createChannelEnumerator(); readEnds = new ArrayList<ReadEnd>(); }
public SerializedUpdateBuffer(List<MemorySegment> memSegments, int segmentSize, IOManager ioManager) { super(memSegments.remove(memSegments.size() - 1), segmentSize, HEADER_LENGTH); totalNumBuffers = memSegments.size() + 1; if (totalNumBuffers < 3) { throw new IllegalArgumentException("SerializedUpdateBuffer needs at least 3 memory segments."); } emptyBuffers = new LinkedBlockingQueue<MemorySegment>(totalNumBuffers); fullBuffers = new ArrayDeque<MemorySegment>(64); emptyBuffers.addAll(memSegments); int threshold = (int) ((1 - SPILL_THRESHOLD) * totalNumBuffers); numSegmentsSpillingThreshold = threshold > 0 ? threshold : 0; minBuffersForWriteEnd = Math.max(2, Math.min(16, totalNumBuffers / 2)); minBuffersForSpilledReadEnd = Math.max(1, Math.min(16, totalNumBuffers / 4)); if (minBuffersForSpilledReadEnd + minBuffersForWriteEnd > totalNumBuffers) { throw new IllegalArgumentException("BUG: Unfulfillable memory assignment."); } this.ioManager = ioManager; channelEnumerator = ioManager.createChannelEnumerator(); readEnds = new ArrayList<ReadEnd>(); }
private void createPartitions(int numPartitions, int recursionLevel) { ensureNumBuffersReturned(numPartitions); this.currentEnumerator = this.ioManager.createChannelEnumerator(); this.partitionsBeingBuilt.clear(); double numRecordPerPartition = (double) buildRowCount / numPartitions; int maxBuffer = maxInitBufferOfBucketArea(numPartitions); for (int i = 0; i < numPartitions; i++) { LongHashPartition p = new LongHashPartition(this, i, buildSideSerializer, numRecordPerPartition, maxBuffer, recursionLevel); this.partitionsBeingBuilt.add(p); } }
private void createPartitions(int numPartitions, int recursionLevel) { // sanity check ensureNumBuffersReturned(numPartitions); this.currentEnumerator = this.ioManager.createChannelEnumerator(); this.partitionsBeingBuilt.clear(); double numRecordPerPartition = (double) buildRowCount / numPartitions; int maxBuffer = maxInitBufferOfBucketArea(numPartitions); for (int i = 0; i < numPartitions; i++) { BinaryHashBucketArea area = new BinaryHashBucketArea(this, numRecordPerPartition, maxBuffer); BinaryHashPartition p = new BinaryHashPartition(area, this.binaryBuildSideSerializer, this.binaryProbeSideSerializer, i, recursionLevel, getNotNullNextBuffer(), this, this.segmentSize, compressionEnable, compressionCodecFactory, compressionBlockSize); area.setPartition(p); this.partitionsBeingBuilt.add(p); } }
ioManager.createChannelEnumerator(), keySerializerFactory.getSerializer(), ioManager); SortedDataFileMerger<Tuple> sortedDataFileMerger = new RecordComparisonMerger<>( blockSortedDataFileFactory, ioManager, keySerializerFactory.getSerializer(), keyComparator,
this.ioManager.createChannelEnumerator();
this.compressionBlockSize = conf.getInteger(TableConfigOptions.SQL_EXEC_SPILL_COMPRESSION_BLOCK_SIZE); this.ioManager = ioManager; this.enumerator = this.ioManager.createChannelEnumerator(); this.channelManager = new SpillChannelManager(); this.merger = new BinaryKVExternalMerger(
getEnvironment().getIOManager().createChannelEnumerator(), this.inputTypeSerializerFactory.getSerializer(), getEnvironment().getIOManager());
case SORT: SortedDataFileFactory<?> sortedDataFileFactory = new BlockSortedDataFileFactory<>( getIOManager().createChannelEnumerator(), this.inputSerializers[inputNum].getSerializer(), getIOManager()); @SuppressWarnings({ "rawtypes", "unchecked" }) getIOManager().createChannelEnumerator(), this.inputSerializers[inputNum].getSerializer(), getIOManager()); @SuppressWarnings({ "rawtypes", "unchecked" })