public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
shuffledMapsCounter, reduceShuffleBytes, failedShuffleCounter); merger = new MergeManager<K, V>(reduceId, jobConf, localFS, localDirAllocator, reporter, codec, combinerClass, combineCollector,
int fetcher ) throws IOException { if (!canShuffleToMemory(requestedSize)) { LOG.info(mapId + ": Shuffling to disk since " + requestedSize + " is greater than maxSingleShuffleLimit (" + usedMemory + ") is lesser than memoryLimit (" + memoryLimit + ")"); return unconditionalReserve(mapId, requestedSize, true);
public void commit() throws IOException { if (type == Type.MEMORY) { merger.closeInMemoryFile(this); } else if (type == Type.DISK) { localFS.rename(tmpOutputPath, outputPath); merger.closeOnDiskFile(outputPath); } else { throw new IOException("Cannot commit MapOutput of type WAIT!"); } }
MapOutput<K, V> mapOutput = merger.reserve(mapTaskId, decompressedLength, id);
kvIter = merger.close(); } catch (Throwable e) { throw new ShuffleError("Error while doing final merge " , e);
MapOutput<K, V> mapOutput = merger.reserve(mapTaskId, decompressedLength, id);
kvIter = merger.close(); } catch (Throwable e) { throw new ShuffleError("Error while doing final merge " , e);
MapOutput<K, V> mapOutput = merger.reserve(mapTaskId, decompressedLength, id);
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
kvIter = merger.close(); } catch (Throwable e) { throw new ShuffleError("Error while doing final merge " , e);
mapOutput = merger.reserve(mapId, decompressedLength, id); } catch (IOException ioe) {
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
kvIter = merger.close(); } catch (Throwable e) { throw new ShuffleError("Error while doing final merge " , e);
MapOutput<K, V> mapOutput = merger.reserve(mapTaskId, decompressedLength, id);
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
kvIter = merger.close(); } catch (Throwable e) { throw new ShuffleError("Error while doing final merge " , e);
mapOutput = merger.reserve(mapId, decompressedLength, id);
public void run() { try { while (!stopped && !Thread.currentThread().isInterrupted()) { MapHost host = null; try { // If merge is on, block merger.waitForResource(); // Get a host to shuffle from host = scheduler.getHost(); metrics.threadBusy(); // Shuffle copyFromHost(host); } finally { if (host != null) { scheduler.freeHost(host); metrics.threadFree(); } } } } catch (InterruptedException ie) { return; } catch (Throwable t) { exceptionReporter.reportException(t); } }
mapOutput = merger.reserve(mapId, decompressedLength, id); } catch (IOException ioe) {