@Override public void close() { if (this.blockIt != null) { try { this.blockIt.close(); } catch (Throwable t) { LOG.error("Error closing block memory iterator: " + t.getMessage(), t); } } this.memoryManager.release(this.memoryForSpillingIterator); }
@Override public T next() { if (this.nextElement == null) { if (!hasNext()) { throw new NoSuchElementException(); } } T out = this.nextElement; this.nextElement = null; return out; }
public void reopen(Iterator<T> input) throws IOException { this.input = input; this.noMoreBlocks = false; this.closed = false; nextBlock(); }
this.blockIt.reopen(blockVals); while (this.blockIt.hasNext()) { final T2 nextBlockRec = this.blockIt.next(); copy1 = this.createCopy(serializer1, firstV1, this.copy1); joinFunction.join(copy1, nextBlockRec, collector); this.blockIt.reset(); final boolean spillingRequired = this.blockIt.hasFurtherInput(); if (spillingRequired) { while (this.blockIt.hasNext()) { T2 nextBlockRec = this.blockIt.next(); this.blockIt.reset(); while (this.blockIt.nextBlock()) { while (this.blockIt.hasNext()) { copy1 = this.createCopy(serializer1, spillHeadCopy, this.copy1); final T2 nextBlockVal = blockIt.next(); joinFunction.join(copy1, nextBlockVal, collector); this.blockIt.reset(); while (this.blockIt.hasNext()) { final T2 nextBlockVal = this.blockIt.next();
if (this.readPhase) { T tmp = getNextRecord(); if (tmp != null) { this.nextElement = tmp; if (this.input.hasNext()) { final T next = this.input.next(); if (writeNextRecord(next)) { this.nextElement = next; return true;
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
if (!writeNextRecord(next)) { throw new IOException("BlockResettableIterator could not serialize record into fresh memory block: " + "Record is too large.");
this.blockIt.reopen(blockVals); while (this.blockIt.hasNext()) { final T2 nextBlockRec = this.blockIt.next(); copy1 = this.createCopy(serializer1, firstV1, this.copy1); joinFunction.join(copy1, nextBlockRec, collector); this.blockIt.reset(); final boolean spillingRequired = this.blockIt.hasFurtherInput(); if (spillingRequired) { while (this.blockIt.hasNext()) { T2 nextBlockRec = this.blockIt.next(); this.blockIt.reset(); while (this.blockIt.nextBlock()) { while (this.blockIt.hasNext()) { copy1 = this.createCopy(serializer1, spillHeadCopy, this.copy1); final T2 nextBlockVal = blockIt.next(); joinFunction.join(copy1, nextBlockVal, collector); this.blockIt.reset(); while (this.blockIt.hasNext()) { final T2 nextBlockVal = this.blockIt.next();
if (this.readPhase) { T tmp = getNextRecord(); if (tmp != null) { this.nextElement = tmp; if (this.input.hasNext()) { final T next = this.input.next(); if (writeNextRecord(next)) { this.nextElement = next; return true;
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
if (!writeNextRecord(next)) { throw new IOException("BlockResettableIterator could not serialize record into fresh memory block: " + "Record is too large.");
this.blockIt.reopen(blockVals); while (this.blockIt.hasNext()) { final T2 nextBlockRec = this.blockIt.next(); copy1 = this.createCopy(serializer1, firstV1, this.copy1); joinFunction.join(copy1, nextBlockRec, collector); this.blockIt.reset(); final boolean spillingRequired = this.blockIt.hasFurtherInput(); if (spillingRequired) { while (this.blockIt.hasNext()) { T2 nextBlockRec = this.blockIt.next(); this.blockIt.reset(); while (this.blockIt.nextBlock()) { while (this.blockIt.hasNext()) { copy1 = this.createCopy(serializer1, spillHeadCopy, this.copy1); final T2 nextBlockVal = blockIt.next(); joinFunction.join(copy1, nextBlockVal, collector); this.blockIt.reset(); while (this.blockIt.hasNext()) { final T2 nextBlockVal = this.blockIt.next();
if (this.readPhase) { T tmp = getNextRecord(); if (tmp != null) { this.nextElement = tmp; if (this.input.hasNext()) { final T next = this.input.next(); if (writeNextRecord(next)) { this.nextElement = next; return true;
@Override public void close() { if (this.blockIt != null) { try { this.blockIt.close(); } catch (Throwable t) { LOG.error("Error closing block memory iterator: " + t.getMessage(), t); } } this.memoryManager.release(this.memoryForSpillingIterator); }
public void reopen(Iterator<T> input) throws IOException { this.input = input; this.noMoreBlocks = false; this.closed = false; nextBlock(); }
@Override public T next() { if (this.nextElement == null) { if (!hasNext()) { throw new NoSuchElementException(); } } T out = this.nextElement; this.nextElement = null; return out; }
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
if (!writeNextRecord(next)) { throw new IOException("BlockResettableIterator could not serialize record into fresh memory block: " + "Record is too large.");
this.blockIt.reopen(blockVals); while (this.blockIt.hasNext()) { final T2 nextBlockRec = this.blockIt.next(); copy1 = this.createCopy(serializer1, firstV1, this.copy1); joinFunction.join(copy1, nextBlockRec, collector); this.blockIt.reset(); final boolean spillingRequired = this.blockIt.hasFurtherInput(); if (spillingRequired) { while (this.blockIt.hasNext()) { T2 nextBlockRec = this.blockIt.next(); this.blockIt.reset(); while (this.blockIt.nextBlock()) { while (this.blockIt.hasNext()) { copy1 = this.createCopy(serializer1, spillHeadCopy, this.copy1); final T2 nextBlockVal = blockIt.next(); joinFunction.join(copy1, nextBlockVal, collector); this.blockIt.reset(); while (this.blockIt.hasNext()) { final T2 nextBlockVal = this.blockIt.next();
if (this.readPhase) { T tmp = getNextRecord(); if (tmp != null) { this.nextElement = tmp; if (this.input.hasNext()) { final T next = this.input.next(); if (writeNextRecord(next)) { this.nextElement = next; return true;