private Path renameCompletedFile(Path file) throws IOException { String fileName = file.toString(); String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix)); String newName = new Path(fileNameMinusSuffix).getName(); Path newFile = new Path(archiveDirPath + Path.SEPARATOR + newName); LOG.info("Completed consuming file {}", fileNameMinusSuffix); if (!hdfs.rename(file, newFile)) { throw new IOException("Rename failed for file: " + file); } LOG.debug("Renamed file {} to {} ", file, newFile); return newFile; }
public static boolean rename(FileSystem fs, Path sourcePath, Path destPath, Configuration conf) throws IOException { LOG.info("Renaming " + sourcePath + " to " + destPath); // If destPath directory exists, rename call will move the sourcePath // into destPath without failing. So check it before renaming. if (fs.exists(destPath)) { throw new IOException("Cannot rename the source path. The destination " + "path already exists."); } return fs.rename(sourcePath, destPath); }
/** Renames an existing map directory. */ public static void rename(FileSystem fs, String oldName, String newName) throws IOException { Path oldDir = new Path(oldName); Path newDir = new Path(newName); if (!fs.rename(oldDir, newDir)) { throw new IOException("Could not rename " + oldDir + " to " + newDir); } }
@Override public void close() throws IOException { // close writer if (outWriter == null) { return; } outWriter.close(); outWriter = null; if (!exception) { FileStatus fss = fs.getFileStatus(outPath); LOG.info("renamed path " + outPath + " to " + finalPath + " . File size is " + fss.getLen()); if (!fs.rename(outPath, finalPath)) { throw new IOException("Unable to rename output to " + finalPath); } } else { if (!autoDelete) { fs.delete(outPath, true); } } }
private void renameOrFail(Path src, Path dest) throws IOException { if (!fs.rename(src, dest)) { throw new IOException("Rename unsuccessful : " + String.format("'%s' to '%s'", src, dest)); } }
/** * Rename bucketPath file from .tmp to permanent location. */ private void renameBucket() throws IOException { Path srcPath = new Path(bucketPath + IN_USE_EXT); Path dstPath = new Path(bucketPath + AVRO_EXT); if (fileSystem.exists(srcPath)) { // could block LOG.info("Renaming " + srcPath + " to " + dstPath); fileSystem.rename(srcPath, dstPath); // could block } }
private static void saveState (FileSystem fs, Path dir, State state) throws IOException { Path tmpFile = new Path(dir, STATE_FILE + ".tmp"); Path newFile = new Path(dir, STATE_FILE); fs.delete(tmpFile, false); try (DataOutputStream dataOutputStream = new DataOutputStream(fs.create(new Path(dir, STATE_FILE + ".tmp")))) { state.write(dataOutputStream); } // Caution: We are deleting right before renaming because rename doesn't support atomic overwrite options from FileSystem API. fs.delete(newFile, false); fs.rename(tmpFile, newFile); }
public void rollback() { if (hasRollback) return; hasRollback = true; try { logger.error("Rollback {} from {}", realPath, hasOldFile ? backPath.toString() : "<empty>"); if (fs.exists(realPath)) fs.delete(realPath, true); if (hasOldFile) fs.rename(backPath, realPath); } catch (IOException ex2) { logger.error("Rollback failed", ex2); } }
private void updateIndex(long txId) { LOG.debug("Starting index update."); final Path tmpPath = tmpFilePath(indexFilePath.toString()); try (FSDataOutputStream out = this.options.fs.create(tmpPath, true); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) { TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset()); bw.write(txnRecord.toString()); bw.newLine(); bw.flush(); out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://), the output stream must be closed before the file associated with it is deleted. For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */ /* * Delete the current index file and rename the tmp file to atomically * replace the index file. Orphan .tmp files are handled in getTxnRecord. */ options.fs.delete(this.indexFilePath, false); options.fs.rename(tmpPath, this.indexFilePath); lastSeenTxn = txnRecord; LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("Begin commit failed due to IOException. Failing batch", e); throw new FailedException(e); } }
private void fsMove(Path from, Path to) throws IOException { if (PartitionUtils.isUnixTimeStamp(from.getName())) { this.versionOwnerFs.rename(from, to.getParent()); } else { for (FileStatus fileStatus : this.versionOwnerFs.listStatus(from)) { if (fileStatus.isFile()) { this.versionOwnerFs.rename(fileStatus.getPath(), to); } } } } }
public static void renameFile(FileSystem fs, Path src, Path dst) throws IOException { if (fs.exists(dst) && !fs.delete(dst, false)) { throw new IOException("Can not delete " + dst); } if (!fs.rename(src, dst)) { throw new IOException("Can not rename from " + src + " to " + dst); } }
private void rename(Path src, Path dst, String peerId) throws IOException { if (fs.exists(src)) { deleteDir(dst, peerId); if (!fs.rename(src, dst)) { throw new IOException( "Failed to rename dir from " + src + " to " + dst + " for peer id=" + peerId); } LOG.info("Renamed dir from {} to {} for peer id={}", src, dst, peerId); } else if (!fs.exists(dst)) { throw new IOException( "Want to rename from " + src + " to " + dst + ", but they both do not exist"); } }
private void corruptLog(final FileStatus logFile, final long dropBytes) throws IOException { assertTrue(logFile.getLen() > dropBytes); LOG.debug("corrupt log " + logFile.getPath() + " size=" + logFile.getLen() + " drop=" + dropBytes); Path tmpPath = new Path(testDir, "corrupted.log"); InputStream in = fs.open(logFile.getPath()); OutputStream out = fs.create(tmpPath); IOUtils.copyBytes(in, out, logFile.getLen() - dropBytes, true); if (!fs.rename(tmpPath, logFile.getPath())) { throw new IOException("Unable to rename"); } }
@Override public void close() throws IOException { // close writer if (outWriter == null) { return; } outWriter.close(); outWriter = null; if (!exception) { FileStatus fss = fs.getFileStatus(outPath); LOG.info("renamed path " + outPath + " to " + finalPath + " . File size is " + fss.getLen()); if (!fs.rename(outPath, finalPath)) { throw new IOException("Unable to rename output to " + finalPath); } } else { if (!autoDelete) { fs.delete(outPath, true); } } }
@Override public void commit() throws IOException { final Path src = recoverable.tempFile(); final Path dest = recoverable.targetFile(); final long expectedLength = recoverable.offset(); final FileStatus srcStatus; try { srcStatus = fs.getFileStatus(src); } catch (IOException e) { throw new IOException("Cannot clean commit: Staging file does not exist."); } if (srcStatus.getLen() != expectedLength) { // something was done to this file since the committer was created. // this is not the "clean" case throw new IOException("Cannot clean commit: File has trailing junk data."); } try { fs.rename(src, dest); } catch (IOException e) { throw new IOException("Committing file by rename failed: " + src + " to " + dest, e); } }
private void handlePendingFilesForPreviousCheckpoints(Map<Long, List<String>> pendingFilesPerCheckpoint) { // Move files that are confirmed by a checkpoint but did not get moved to final location // because the checkpoint notification did not happen before a failure LOG.debug("Moving pending files to final location on restore."); Set<Long> pastCheckpointIds = pendingFilesPerCheckpoint.keySet(); for (Long pastCheckpointId : pastCheckpointIds) { // All the pending files are buckets that have been completed but are waiting to be renamed // to their final name for (String filename : pendingFilesPerCheckpoint.get(pastCheckpointId)) { Path finalPath = new Path(filename); Path pendingPath = getPendingPathFor(finalPath); try { if (fs.exists(pendingPath)) { LOG.debug("Restoring BucketingSink State: Moving pending file {} to final location after complete checkpoint {}.", pendingPath, pastCheckpointId); fs.rename(pendingPath, finalPath); } } catch (IOException e) { LOG.error("Restoring BucketingSink State: Error while renaming pending file {} to final path {}: {}", pendingPath, finalPath, e); throw new RuntimeException("Error while renaming pending file " + pendingPath + " to final path " + finalPath, e); } } } }
private static void renameHDFSPath(String srcPath, String dstPath) throws IOException, InterruptedException { int nRetry = 0; int sleepTime = 5000; while (!hdfsFS.rename(new Path(srcPath), new Path(dstPath))) { ++nRetry; if (nRetry > 3) { throw new InterruptedException("Cannot rename folder " + srcPath + " to folder " + dstPath); } else { Thread.sleep(sleepTime * nRetry * nRetry); } } } }