/** Get a path from the local FS. This method should be used if the size of * the file is not known apriori. We go round-robin over the set of disks * (via the configured dirs) and return the first complete path where * we could create the parent directory of the passed path. * @param pathStr the requested path (this will be created on the first * available disk) * @param conf the Configuration object * @return the complete path to the file on a local disk * @throws IOException */ public Path getLocalPathForWrite(String pathStr, Configuration conf) throws IOException { return getLocalPathForWrite(pathStr, SIZE_UNKNOWN, conf); }
/** Get a path from the local FS. Pass size as * SIZE_UNKNOWN if not known apriori. We * round-robin over the set of disks (via the configured dirs) and return * the first complete path which has enough space * @param pathStr the requested path (this will be created on the first * available disk) * @param size the size of the file that is going to be written * @param conf the Configuration object * @return the complete path to the file on a local disk * @throws IOException */ public Path getLocalPathForWrite(String pathStr, long size, Configuration conf) throws IOException { return getLocalPathForWrite(pathStr, size, conf, true); }
new Path(tmpDir, "intermediate").suffix("." + passNo); Path outputFile = lDirAlloc.getLocalPathForWrite( tmpFilename.toString(), approxOutputSize, conf);
/** * Create a local map output file name. * * @param size the size of the file * @return path * @throws IOException */ public Path getOutputFileForWrite(long size) throws IOException { return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR + "file.out", size, conf); }
public Path getLogPathForWrite(String pathStr, boolean checkWrite) throws IOException { return logDirsAllocator.getLocalPathForWrite(pathStr, LocalDirAllocator.SIZE_UNKNOWN, getConfig(), checkWrite); }
public Path getLocalPathForWrite(String pathStr, long size, boolean checkWrite) throws IOException { return localDirsAllocator.getLocalPathForWrite(pathStr, size, getConfig(), checkWrite); }
/** * Create a local map output file name. * * @param size the size of the file * @return path * @throws IOException */ public Path getOutputFileForWrite(long size) throws IOException { Path attemptOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf); }
public Path getLogPathForWrite(String pathStr, boolean checkWrite) throws IOException { return logDirsAllocator.getLocalPathForWrite(pathStr, LocalDirAllocator.SIZE_UNKNOWN, getConfig(), checkWrite); }
/** * Create a local map output file name. * * @param size the size of the file * @return path * @throws IOException */ public Path getOutputFileForWrite(long size) throws IOException { Path attemptOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf); }
/** * Get a temporary path. * @param conf configuration to use when creating the allocator * @param prefix filename prefix * @param size file size, or -1 if not known * @return the temp path. * @throws IOException IO failure */ public static Path tempPath(Configuration conf, String prefix, long size) throws IOException { return getAllocator(conf, BUFFER_DIR) .getLocalPathForWrite(prefix, size, conf); }
/** * Create a local map output file name. * * @param size the size of the file * @return path * @throws IOException */ public Path getOutputFileForWrite(long size) throws IOException { Path attemptOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf); }
public Path getLogPathForWrite(String pathStr, boolean checkWrite) throws IOException { return logDirsAllocator.getLocalPathForWrite(pathStr, LocalDirAllocator.SIZE_UNKNOWN, getConfig(), checkWrite); }
public Path createBaseDir() throws IOException { // the base dir for an output dir String baseDir = getBaseOutputDir(executionBlockId).toString(); Path baseDirPath = localFS.makeQualified(getLocalDirAllocator().getLocalPathForWrite(baseDir, systemConf)); return baseDirPath; }
/** * Get a local path from all temporal paths in round-robin manner. */ private synchronized Path getChunkPathForWrite(int level, int chunkId) throws IOException { return localFS.makeQualified(localDirAllocator.getLocalPathForWrite( sortTmpDir + "/" + level + "_" + chunkId, context.getConf())); }
/** * Get a local path from all temporal paths in round-robin manner. */ private synchronized Path getChunkPathForWrite(int level, int chunkId) throws IOException { return localFS.makeQualified(localDirAllocator.getLocalPathForWrite( sortTmpDir + "/" + level + "_" + chunkId, context.getConf())); }
/** Create a local map output index file name. * @param mapTaskId a map task id * @param size the size of the file */ public Path getOutputIndexFileForWrite(TaskAttemptID mapTaskId, long size) throws IOException { return lDirAlloc.getLocalPathForWrite(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/file.out.index", size, conf); }
/** Create a local map output file name. * @param mapTaskId a map task id * @param size the size of the file */ public Path getOutputFileForWrite(TaskAttemptID mapTaskId, long size) throws IOException { return lDirAlloc.getLocalPathForWrite(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/file.out", size, conf); }
/** Create a local map output index file name. * @param mapTaskId a map task id * @param size the size of the file */ public Path getOutputIndexFileForWrite(TaskAttemptID mapTaskId, long size) throws IOException { return lDirAlloc.getLocalPathForWrite(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/file.out.index", size, conf); }
/** Create a local map spill file name. * @param mapTaskId a map task id * @param spillNumber the number * @param size the size of the file */ public Path getSpillFileForWrite(TaskAttemptID mapTaskId, int spillNumber, long size) throws IOException { return lDirAlloc.getLocalPathForWrite(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/spill" + spillNumber + ".out", size, conf); }
/** Create a local map spill index file name. * @param mapTaskId a map task id * @param spillNumber the number * @param size the size of the file */ public Path getSpillIndexFileForWrite(TaskAttemptID mapTaskId, int spillNumber, long size) throws IOException { return lDirAlloc.getLocalPathForWrite(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/spill" + spillNumber + ".out.index", size, conf); }