private String getPartitionRootLocation(String ptnLocn, int numPtnKeys) { if (customDynamicLocationUsed) { return null; } if (ptnRootLocation == null) { // we only need to calculate it once, it'll be the same for other partitions in this job. Path ptnRoot = new Path(ptnLocn); for (int i = 0; i < numPtnKeys; i++) { // LOG.info("Getting parent of "+ptnRoot.getName()); ptnRoot = ptnRoot.getParent(); } ptnRootLocation = ptnRoot.toString(); } // LOG.info("Returning final parent : "+ptnRootLocation); return ptnRootLocation; }
FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); new Path(getTableBackupDir(backupInfo.getBackupRootDir(), backupInfo.getBackupId(), table)); if (outputFs.delete(targetDirPath, true)) { LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); } else { LOG.info("No data has been found in " + targetDirPath.toString() + "."); Path tableDir = targetDirPath.getParent(); FileStatus[] backups = listStatus(outputFs, tableDir, null); if (backups == null || backups.length == 0) { outputFs.delete(tableDir, true); LOG.debug(tableDir.toString() + " is empty, remove it."); outputFs.delete(new Path(targetDir, backupInfo.getBackupId()), true); } catch (IOException e1) { LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " at "
private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException { URI uri = hbaseRoot.toUri().relativize(path.toUri()); if (uri.isAbsolute()) return false; String relativePath = uri.getPath(); Path rootDir = getSidelineDir(); Path dst = new Path(rootDir, relativePath); boolean pathCreated = fs.mkdirs(dst.getParent()); if (!pathCreated) { LOG.error("Failed to create path: " + dst.getParent()); return false; } LOG.info("Trying to sideline file " + path + " to " + dst); return fs.rename(path, dst); }
private void moveDirectory(String sourceDir, String targetDir) throws IOException { // If targetDir exists, delete it if (this.fs.exists(new Path(targetDir))) { deleteDirectory(targetDir); } // Create parent directories of targetDir WriterUtils.mkdirsWithRecursivePermission(this.fs, new Path(targetDir).getParent(), FsPermission.getCachePoolDefault()); // Move directory log.info("Moving directory: " + sourceDir + " to: " + targetDir); if (!this.fs.rename(new Path(sourceDir), new Path(targetDir))) { throw new IOException(String.format("Unable to move %s to %s", sourceDir, targetDir)); } }
private void createRecordAndCorruptMobFile(TableName tn, byte[] row, byte[] family, byte[] qf, byte[] value) throws IOException { Put put1 = new Put(row); put1.addColumn(family, qf, value); table.put(put1); admin.flush(tn); Path mobFile = getFlushedMobFile(conf, fs, tn, Bytes.toString(family)); Assert.assertNotNull(mobFile); // create new corrupt mob file. Path corruptFile = new Path(mobFile.getParent(), "dummy"); TestHFile.truncateFile(fs, mobFile, corruptFile); fs.delete(mobFile, true); fs.rename(corruptFile, mobFile); }
/** * Get all the unrenamed directories from the given paths * They are deepest level containing directories whose name doesn't have a suffix {@link MRCompactor#COMPACTION_RENAME_SOURCE_DIR_SUFFIX} * Also each directory needs to contain at least one file so empty directories will be excluded from the result */ public static Set<Path> getDeepestLevelUnrenamedDirsWithFileExistence (FileSystem fs, Set<Path> paths) throws IOException { Set<Path> unrenamed = Sets.newHashSet(); for (FileStatus fileStatus : FileListUtils.listFilesRecursively(fs, paths)) { if (!fileStatus.getPath().getParent().toString().endsWith(MRCompactor.COMPACTION_RENAME_SOURCE_DIR_SUFFIX)) { unrenamed.add(fileStatus.getPath().getParent()); } } return unrenamed; }
/** * Open a BufferedWriter * @param errFilePath path to write the file */ public void open(Path errFilePath) throws IOException { this.fs.mkdirs(errFilePath.getParent()); OutputStream os = this.closer.register(this.fs.exists(errFilePath) ? this.fs.append(errFilePath) : this.fs.create(errFilePath)); this.writer = this.closer .register(new BufferedWriter(new OutputStreamWriter(os, ConfigurationKeys.DEFAULT_CHARSET_ENCODING))); }
@Override public void commit() throws IOException { checkIsNotTmp(); // FileContext supports atomic rename, whereas FileSystem doesn't FileContext fc = FileContext.getFileContext(_hadoopConf); Path dest = new Path(_path.getParent(), BLOBSTORE_DATA_FILE); if (_mustBeNew) { fc.rename(_path, dest); } else { fc.rename(_path, dest, Options.Rename.OVERWRITE); } // Note, we could add support for setting the replication factor }
public String getStagingPartitionLocation() { Path originalPartitionLocation = getLocation(); if (PartitionUtils.isUnixTimeStamp(originalPartitionLocation.getName())) { return StringUtils.join(Arrays.asList(getLocation().getParent().toString(), this.timeStamp), '/'); } else { return StringUtils.join(Arrays.asList(getLocation().toString(), this.timeStamp), '/'); } }
public static void deletePathAndEmptyAncestors(FileSystem fs, Path f, boolean recursive) throws IOException { deletePath(fs, f, recursive); Path parent = f.getParent(); while (parent != null) { if (fs.exists(parent) && fs.listStatus(parent).length == 0) { deletePath(fs, parent, true); parent = parent.getParent(); } else { break; } } }
private void createDatasetFiles() throws IOException { // Create writer output files Path datasetWriterOutputPath = new Path(writerOutputPath, copyEntity.getDatasetAndPartition(this.metadata).identifier()); Path outputPathWithCurrentDirectory = new Path(datasetWriterOutputPath, PathUtils.withoutLeadingSeparator(this.targetPath)); for (String path : relativeFilePaths) { Path pathToCreate = new Path(outputPathWithCurrentDirectory, path); fs.mkdirs(pathToCreate.getParent()); fs.create(pathToCreate); } }
/** * Returns the latest modified file at the uri of interest. * * @param uri Either a directory or a file on HDFS. If it is a file, the parent directory will be searched. * @param pattern A pattern matcher for file names in the directory of interest. Passing `null` results in matching any file in the directory. * * @return The URI of the file with the most recent modified timestamp. */ @Override public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) { final Path path = new Path(uri); try { return RetryUtils.retry( () -> { final FileSystem fs = path.getFileSystem(config); if (!fs.exists(path)) { return null; } return mostRecentInDir(fs.isDirectory(path) ? path : path.getParent(), pattern); }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT ); } catch (Exception e) { throw Throwables.propagate(e); } }