/** * Recursively lists status for all files starting from a particular directory (or individual file * as base case). * * @param fs * file system * * @param fileStatus * starting point in file system * * @param results * receives enumeration of all files found */ public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, List<FileStatus> results) throws IOException { listStatusRecursively(fs, fileStatus, HIDDEN_FILES_PATH_FILTER, results); }
public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, PathFilter filter, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath(), filter)) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
/** * Recursively lists status for all files starting from the directory dir * @param job * @param dir * @return * @throws IOException */ protected FileStatus[] listStatus(JobConf job, Path dir) throws IOException { ArrayList<FileStatus> result = new ArrayList<FileStatus>(); List<IOException> errors = new ArrayList<IOException>(); FileSystem fs = dir.getFileSystem(job); FileStatus[] matches = fs.globStatus(dir, FileUtils.HIDDEN_FILES_PATH_FILTER); if (matches == null) { errors.add(new IOException("Input path does not exist: " + dir)); } else if (matches.length == 0) { errors.add(new IOException("Input Pattern " + dir + " matches 0 files")); } else { for (FileStatus globStat : matches) { FileUtils.listStatusRecursively(fs, globStat, result); } } if (!errors.isEmpty()) { throw new InvalidInputException(errors); } LOG.debug("Matches for " + dir + ": " + result); LOG.info("Total input paths to process : " + result.size() + " from dir " + dir); return result.toArray(new FileStatus[result.size()]); }
/** * Recursively lists status for all files starting from the directory dir * @param job * @param dir * @return * @throws IOException */ protected FileStatus[] listStatus(JobConf job, Path dir) throws IOException { ArrayList<FileStatus> result = new ArrayList<FileStatus>(); List<IOException> errors = new ArrayList<IOException>(); FileSystem fs = dir.getFileSystem(job); FileStatus[] matches = fs.globStatus(dir, FileUtils.HIDDEN_FILES_PATH_FILTER); if (matches == null) { errors.add(new IOException("Input path does not exist: " + dir)); } else if (matches.length == 0) { errors.add(new IOException("Input Pattern " + dir + " matches 0 files")); } else { for (FileStatus globStat : matches) { FileUtils.listStatusRecursively(fs, globStat, result); } } if (!errors.isEmpty()) { throw new InvalidInputException(errors); } LOG.debug("Matches for " + dir + ": " + result); LOG.info("Total input paths to process : " + result.size() + " from dir " + dir); return result.toArray(new FileStatus[result.size()]); }
/** * Lists status for all files under a given path. Whether or not this is recursive depends on the * setting of job configuration parameter mapred.input.dir.recursive. * * @param fs * file system * * @param p * path in file system * * @return list of file status entries */ private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException { boolean recursive = job.getBoolean(FileInputFormat.INPUT_DIR_RECURSIVE, false); // If this is in acid format always read it recursively regardless of what the jobconf says. if (!recursive && !AcidUtils.isAcid(p, job)) { return fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER); } List<FileStatus> results = new ArrayList<FileStatus>(); for (FileStatus stat : fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER)) { FileUtils.listStatusRecursively(fs, stat, results); } return results.toArray(new FileStatus[results.size()]); }
/** * Lists status for all files under a given path. Whether or not this is recursive depends on the * setting of job configuration parameter mapred.input.dir.recursive. * * @param fs * file system * * @param p * path in file system * * @return list of file status entries */ private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException { boolean recursive = job.getBoolean(FileInputFormat.INPUT_DIR_RECURSIVE, false); // If this is in acid format always read it recursively regardless of what the jobconf says. if (!recursive && !AcidUtils.isAcid(p, job)) { return fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER); } List<FileStatus> results = new ArrayList<FileStatus>(); for (FileStatus stat : fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER)) { FileUtils.listStatusRecursively(fs, stat, results); } return results.toArray(new FileStatus[results.size()]); }
try { FileStatus fileStatus = fs.getFileStatus(path); FileUtils.listStatusRecursively(fs, fileStatus, filter, result); } catch (IOException e) {
/** * Recursively lists status for all files starting from a particular directory (or individual file * as base case). * * @param fs * file system * * @param fileStatus * starting point in file system * * @param results * receives enumeration of all files found */ public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, List<FileStatus> results) throws IOException { listStatusRecursively(fs, fileStatus, HIDDEN_FILES_PATH_FILTER, results); }
public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, PathFilter filter, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath(), filter)) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
/** * Recursively lists status for all files starting from a particular directory (or individual file * as base case). * * @param fs * file system * * @param fileStatus * starting point in file system * * @param results * receives enumeration of all files found */ public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath(), HIDDEN_FILES_PATH_FILTER)) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
protected FileStatus[] listStatus(JobConf job, Path path) throws IOException { ArrayList<FileStatus> result = new ArrayList<FileStatus>(); List<IOException> errors = new ArrayList<IOException>(); FileSystem fs = path.getFileSystem(job); FileStatus[] matches = fs.globStatus(path); if (matches == null) { errors.add(new IOException("Input path does not exist: " + path)); } else if (matches.length == 0) { errors.add(new IOException("Input Pattern " + path + " matches 0 files")); } else { for (FileStatus globStat : matches) { FileUtils.listStatusRecursively(fs, globStat, result); } } if (!errors.isEmpty()) { throw new InvalidInputException(errors); } LOG.info("Total input paths to process : " + result.size()); return result.toArray(new FileStatus[result.size()]); }
/** * Recursively lists status for all files starting from a particular directory (or individual file * as base case). * * @param fs * file system * * @param fileStatus * starting point in file system * * @param results * receives enumeration of all files found */ public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath())) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
/** * Recursively lists status for all files starting from a particular directory (or individual file * as base case). * * @param fs * file system * * @param fileStatus * starting point in file system * * @param results * receives enumeration of all files found */ public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath())) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
try { FileStatus fileStatus = fs.getFileStatus(path); FileUtils.listStatusRecursively(fs, fileStatus, result); } catch (IOException e) {
/** * Lists status for all files under a given path. Whether or not this is recursive depends on the * setting of job configuration parameter mapred.input.dir.recursive. * * @param fs * file system * * @param p * path in file system * * @return list of file status entries */ private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException { boolean recursive = HiveConf.getBoolVar(job, HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE); // If this is in acid format always read it recursively regardless of what the jobconf says. if (!recursive && !AcidUtils.isAcid(p, job)) { return fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER); } List<FileStatus> results = new ArrayList<FileStatus>(); for (FileStatus stat : fs.listStatus(p, FileUtils.HIDDEN_FILES_PATH_FILTER)) { FileUtils.listStatusRecursively(fs, stat, results); } return results.toArray(new FileStatus[results.size()]); }
/** * Lists status for all files under a given path. Whether or not this is recursive depends on the * setting of job configuration parameter mapred.input.dir.recursive. * * @param fs * file system * * @param p * path in file system * * @return list of file status entries */ private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException { HiveConf hiveConf = new HiveConf(job, FetchOperator.class); boolean recursive = hiveConf.getBoolVar(HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE); if (!recursive) { return fs.listStatus(p); } List<FileStatus> results = new ArrayList<FileStatus>(); for (FileStatus stat : fs.listStatus(p)) { FileUtils.listStatusRecursively(fs, stat, results); } return results.toArray(new FileStatus[results.size()]); } }
/** * Recursively lists status for all files starting from the directory dir * @param job * @param dir * @return * @throws IOException */ protected FileStatus[] listStatus(JobConf job, Path dir) throws IOException { ArrayList<FileStatus> result = new ArrayList<FileStatus>(); List<IOException> errors = new ArrayList<IOException>(); FileSystem fs = dir.getFileSystem(job); FileStatus[] matches = fs.globStatus(dir, FileUtils.HIDDEN_FILES_PATH_FILTER); if (matches == null) { errors.add(new IOException("Input path does not exist: " + dir)); } else if (matches.length == 0) { errors.add(new IOException("Input Pattern " + dir + " matches 0 files")); } else { for (FileStatus globStat : matches) { FileUtils.listStatusRecursively(fs, globStat, result); } } if (!errors.isEmpty()) { throw new InvalidInputException(errors); } LOG.debug("Matches for " + dir + ": " + result); LOG.info("Total input paths to process : " + result.size() + " from dir " + dir); return result.toArray(new FileStatus[result.size()]); }
try { FileStatus fileStatus = fs.getFileStatus(path); FileUtils.listStatusRecursively(fs, fileStatus, filter, result); } catch (IOException e) {