/** * A wrapper around {@link FileSystem#delete(Path, boolean)} that only deletes a given {@link Path} if it is present * on the given {@link FileSystem}. */ public static void deleteIfExists(FileSystem fs, Path path, boolean recursive) throws IOException { if (fs.exists(path)) { deletePath(fs, path, recursive); } }
public static void deletePathAndEmptyAncestors(FileSystem fs, Path f, boolean recursive) throws IOException { deletePath(fs, f, recursive); Path parent = f.getParent(); while (parent != null) { if (fs.exists(parent) && fs.listStatus(parent).length == 0) { deletePath(fs, parent, true); parent = parent.getParent(); } else { break; } } }
/** * {@inheritDoc}. * * <p> * This default implementation simply deletes the staging file if it exists. * </p> * * @throws IOException if deletion of the staging file fails */ @Override public void cleanup() throws IOException { // Delete the staging file if (this.fs.exists(this.stagingFile)) { HadoopUtils.deletePath(this.fs, this.stagingFile, false); } }
/** * Delete files according to the regular expression provided * @param fs Filesystem object * @param path base path * @param regex regular expression to select files to delete * @throws IOException */ public static void deletePathByRegex(FileSystem fs, final Path path, final String regex) throws IOException { FileStatus[] statusList = fs.listStatus(path, path1 -> path1.getName().matches(regex)); for (final FileStatus oldJobFile : statusList) { HadoopUtils.deletePath(fs, oldJobFile.getPath(), true); } }
@Override public Void call() throws Exception { Lock lock = ParallelRunner.this.locks.get(path.toString()); lock.lock(); try { HadoopUtils.deletePath(ParallelRunner.this.fs, path, recursive); return null; } finally { lock.unlock(); } } }), "Delete path " + path));
/** * Calls deletePath() on each directory in the given list of directories to delete. * If moveToTrash is set, it will be moved to trash according to the file system trash policy. */ public static void deleteDirectories(FileSystem fs, List<String> directoriesToDelete, boolean recursive, boolean moveToTrash) throws IOException { for (String directory : directoriesToDelete) { if (moveToTrash) { moveToTrash(fs, new Path(directory)); } else { deletePath(fs, new Path(directory), recursive); } } }
private void deleteFilesOnSource(WorkUnitState state) throws IOException { CopyEntity copyEntity = CopySource.deserializeCopyEntity(state); if (copyEntity instanceof CopyableFile) { HadoopUtils.deletePath(this.sourceFs, ((CopyableFile) copyEntity).getOrigin().getPath(), true); HadoopUtils.deletePath(this.sourceFs, PathUtils.addExtension(((CopyableFile) copyEntity).getOrigin().getPath(), ReadyCopyableFileFilter.READY_EXTENSION), true); } } }
/** * Publish metadata to a set of paths */ private void publishMetadata(String metadataValue, int branchId, Path metadataOutputPath) throws IOException { try { if (metadataOutputPath == null) { LOG.info("Metadata output path not set for branch " + String.valueOf(branchId) + ", not publishing."); return; } if (metadataValue == null) { LOG.info("No metadata collected for branch " + String.valueOf(branchId) + ", not publishing."); return; } FileSystem fs = this.metaDataWriterFileSystemByBranches.get(branchId); if (!fs.exists(metadataOutputPath.getParent())) { WriterUtils.mkdirsWithRecursivePermissionWithRetry(fs, metadataOutputPath, this.permissions.get(branchId), retrierConfig); } //Delete the file if metadata already exists if (fs.exists(metadataOutputPath)) { HadoopUtils.deletePath(fs, metadataOutputPath, false); } LOG.info("Writing metadata for branch " + String.valueOf(branchId) + " to " + metadataOutputPath.toString()); try (FSDataOutputStream outputStream = fs.create(metadataOutputPath)) { outputStream.write(metadataValue.getBytes(StandardCharsets.UTF_8)); } } catch (IOException e) { LOG.error("Metadata file is not generated: " + e, e); } }
public static void cleanUpOldJobData(State state, Logger logger, boolean stagingDirProvided, boolean outputDirProvided) throws IOException { Set<Path> jobPaths = new HashSet<>(); String writerFsUri = state.getProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, ConfigurationKeys.LOCAL_FS_URI); FileSystem fs = FileSystem.get(URI.create(writerFsUri), WriterUtils.getFsConfiguration(state)); Path jobPath; if (stagingDirProvided) { jobPath = new Path(state.getProp(ConfigurationKeys.WRITER_STAGING_DIR)).getParent(); } else { jobPath = new Path(state.getProp(ConfigurationKeys.WRITER_STAGING_DIR)).getParent().getParent(); } jobPaths.add(jobPath); if (outputDirProvided) { jobPath = new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR)).getParent(); } else { jobPath = new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR)).getParent().getParent(); } jobPaths.add(jobPath); for (Path jobPathToDelete : jobPaths) { logger.info("Cleaning up old job directory " + jobPathToDelete); HadoopUtils.deletePath(fs, jobPathToDelete, true); } } /**
@Override public Void call() throws Exception { Configuration conf = new Configuration(ParallelRunner.this.fs.getConf()); WritableShimSerialization.addToHadoopConfiguration(conf); try (@SuppressWarnings("deprecation") SequenceFile.Reader reader = new SequenceFile.Reader( ParallelRunner.this.fs, inputFilePath, conf)) { Writable key = keyClass.newInstance(); T state = stateClass.newInstance(); while (reader.next(key)) { state = (T) reader.getCurrentValue(state); states.add(state); state = stateClass.newInstance(); } if (deleteAfter) { HadoopUtils.deletePath(ParallelRunner.this.fs, inputFilePath, false); } } return null; } }), "Deserialize state from file " + inputFilePath));
HadoopUtils.deletePath(fs, filePath, true);
HadoopUtils.deletePath(fs, jobStagingPath, true); HadoopUtils.deletePath(fs, jobStagingPath.getParent(), true); HadoopUtils.deletePath(fs, jobOutputPath, true); HadoopUtils.deletePath(fs, jobOutputPath.getParent(), true);
if (overwriteDst && dstFs.exists(dst)) { try { deletePath(dstFs, dst, true); } finally { renamePath(dstFs, tmp, dst); deletePath(dstFs, tmp, true);
HadoopUtils.deletePath(this.fs, this.stagingFile, false);
HadoopUtils.deletePath(this.fs, new Path(sourceState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR)), true);
/** * A wrapper around {@link FileSystem#delete(Path, boolean)} that only deletes a given {@link Path} if it is present * on the given {@link FileSystem}. */ public static void deleteIfExists(FileSystem fs, Path path, boolean recursive) throws IOException { if (fs.exists(path)) { deletePath(fs, path, recursive); } }
public static void deletePathAndEmptyAncestors(FileSystem fs, Path f, boolean recursive) throws IOException { deletePath(fs, f, recursive); Path parent = f.getParent(); while (parent != null) { if (fs.exists(parent) && fs.listStatus(parent).length == 0) { deletePath(fs, parent, true); parent = parent.getParent(); } else { break; } } }
@Override public Void call() throws Exception { Lock lock = ParallelRunner.this.locks.get(path.toString()); lock.lock(); try { HadoopUtils.deletePath(ParallelRunner.this.fs, path, recursive); return null; } finally { lock.unlock(); } } }), "Delete path " + path));
/** * Calls deletePath() on each directory in the given list of directories to delete. * If moveToTrash is set, it will be moved to trash according to the file system trash policy. */ public static void deleteDirectories(FileSystem fs, List<String> directoriesToDelete, boolean recursive, boolean moveToTrash) throws IOException { for (String directory : directoriesToDelete) { if (moveToTrash) { moveToTrash(fs, new Path(directory)); } else { deletePath(fs, new Path(directory), recursive); } } }
private void deleteFilesOnSource(WorkUnitState state) throws IOException { CopyEntity copyEntity = CopySource.deserializeCopyEntity(state); if (copyEntity instanceof CopyableFile) { HadoopUtils.deletePath(this.sourceFs, ((CopyableFile) copyEntity).getOrigin().getPath(), true); HadoopUtils.deletePath(this.sourceFs, PathUtils.addExtension(((CopyableFile) copyEntity).getOrigin().getPath(), ReadyCopyableFileFilter.READY_EXTENSION), true); } } }