MutableObjectIterator<String> inputIterator = new StringReaderMutableObjectIterator(reader); sorter = new UnilateralSortMerger<String>(mm, ioMan, inputIterator, new DummyInvokable(), new RuntimeSerializerFactory<String>(serializer, String.class), comparator, 1.0, 4, 0.8f, true /* use large record handler */, false); MutableObjectIterator<String> sortedData = sorter.getIterator(); sorter.close();
this.readThread = getReadingThread(exceptionHandler, input, circularQueues, largeRecordHandler, parentTask, serializer, startSpillingBytes); this.sortThread = getSortingThread(exceptionHandler, circularQueues, parentTask); this.spillThread = getSpillingThread(sortedDataFileFactory, spilledFiles, merger, exceptionHandler, circularQueues, parentTask, memoryManager, ioManager, serializerFactory, comparator, this.sortReadMemory, this.writeMemoryForSpilling, maxNumFileHandles); this.mergingThread = getMergingThread(merger, memoryManager, exceptionHandler, parentTask, spilledFiles); startThreads();
InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer); keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false, this.executionConfig.isObjectReuseEnabled()); result = keySorter.getIterator(); } catch (InterruptedException e) { throw new IOException(e);
this.queues.sort.add(UnilateralSortMerger.<E>spillingMarker()); final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); bytesUntilSpilling = 0; final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); bytesUntilSpilling = 0; final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); final CircularElement<E> EOF_MARKER = endMarker(); this.queues.sort.add(EOF_MARKER); LOG.debug("Reading thread done.");
public void handleException(IOException exception) { // forward exception if (!closed) { setResultIteratorException(exception); close(); } } };
case SORT: @SuppressWarnings({ "rawtypes", "unchecked" }) UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(), this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum), this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
keySorter.close(); keySorter = null; } catch (Throwable t) {
public void handleException(IOException exception) { // forward exception if (!closed) { setResultException(exception); close(); } } };
this.readThread = getReadingThread(exceptionHandler, input, circularQueues, largeRecordHandler, parentTask, serializer, ((long) (startSpillingFraction * sortMemory))); this.sortThread = getSortingThread(exceptionHandler, circularQueues, parentTask); this.spillThread = getSpillingThread(exceptionHandler, circularQueues, parentTask, memoryManager, ioManager, serializerFactory, comparator, this.sortReadMemory, this.writeMemory, maxNumFileHandles); startThreads();
InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer); keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false, this.executionConfig.isObjectReuseEnabled()); result = keySorter.getIterator(); } catch (InterruptedException e) { throw new IOException(e);
this.queues.sort.add(UnilateralSortMerger.<E>spillingMarker()); final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); bytesUntilSpilling = 0; final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); bytesUntilSpilling = 0; final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); final CircularElement<E> EOF_MARKER = endMarker(); this.queues.sort.add(EOF_MARKER); LOG.debug("Reading thread done.");
public void handleException(IOException exception) { // forward exception if (!closed) { setResultIteratorException(exception); close(); } } };
case SORT: @SuppressWarnings({ "rawtypes", "unchecked" }) UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(), this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum), this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
keySorter.close(); keySorter = null; } catch (Throwable t) {
this.readThread = getReadingThread(exceptionHandler, input, circularQueues, largeRecordHandler, parentTask, serializer, ((long) (startSpillingFraction * sortMemory))); this.sortThread = getSortingThread(exceptionHandler, circularQueues, parentTask); this.spillThread = getSpillingThread(exceptionHandler, circularQueues, parentTask, memoryManager, ioManager, serializerFactory, comparator, this.sortReadMemory, this.writeMemory, maxNumFileHandles); startThreads();
MutableObjectIterator<StringValue> inputIterator = new StringValueReaderMutableObjectIterator(reader); sorter = new UnilateralSortMerger<StringValue>(mm, ioMan, inputIterator, new DummyInvokable(), new RuntimeSerializerFactory<StringValue>(serializer, StringValue.class), comparator, 1.0, 4, 0.8f, true /* use large record handler */, true); MutableObjectIterator<StringValue> sortedData = sorter.getIterator(); sorter.close();
InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer); keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false, this.executionConfig.isObjectReuseEnabled()); result = keySorter.getIterator(); } catch (InterruptedException e) { throw new IOException(e);
this.queues.sort.add(UnilateralSortMerger.<E>spillingMarker()); final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); bytesUntilSpilling = 0; final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); bytesUntilSpilling = 0; final CircularElement<E> SPILLING_MARKER = spillingMarker(); this.queues.sort.add(SPILLING_MARKER); final CircularElement<E> EOF_MARKER = endMarker(); this.queues.sort.add(EOF_MARKER); LOG.debug("Reading thread done.");
public void handleException(IOException exception) { // forward exception if (!closed) { setResultIteratorException(exception); close(); } } };
case SORT: @SuppressWarnings({ "rawtypes", "unchecked" }) UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(), this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum), this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),