for (FileStatus stat : getFileSystem().globStatus(new Path(sourcePath,"*.avro"))) FileStatus[] incrementalParts = getFileSystem().globStatus(new Path(sourcePath,"*"), new PathFilter() { @Override public boolean accept(Path path) if (!getFileSystem().exists(parent)) getFileSystem().mkdirs(parent); getFileSystem().rename(stat.getPath(), new Path(parent,stat.getPath().getName())); _log.info(String.format("Moving %s to %s",src.getName(),target)); getFileSystem().mkdirs(target.getParent()); if (!getFileSystem().rename(src, target))
/** * Removes all but the more recent days from the ouput that are within the retention period, if one is specified. * * @throws IOException */ private void applyRetention() throws IOException { if (getRetentionCount() != null) { PathUtils.keepLatestNestedDatedPaths(getFileSystem(), getOutputPath(), getRetentionCount()); } }
PartitionPreservingExecutionPlanner planner = new PartitionPreservingExecutionPlanner(getFileSystem(),getProperties()); planner.setInputPaths(getInputPaths()); planner.setOutputPath(getOutputPath()); if (getFileSystem().exists(counters)) if (getFileSystem().exists(target)) getFileSystem().delete(target, true); getFileSystem().rename(counters, target);
@Override protected void initialize() { _garbage = new FileCleaner(getFileSystem()); if (getMaxIterations() == null) { setMaxIterations(20); } if (getMaxToProcess() == null) { if (getNumDays() != null) { setMaxToProcess(getNumDays()); } else { setMaxToProcess(90); } } super.initialize(); }