public static void mkdirsWithRecursivePermissionWithRetry(final FileSystem fs, final Path path, FsPermission perm, Config retrierConfig) throws IOException { if (fs.exists(path)) { return; } if (path.getParent() != null && !fs.exists(path.getParent())) { mkdirsWithRecursivePermissionWithRetry(fs, path.getParent(), perm, retrierConfig); } if (!fs.mkdirs(path, perm)) { throw new IOException(String.format("Unable to mkdir %s with permission %s", path, perm)); } if (retrierConfig != NO_RETRY_CONFIG) { //Wait until file is not there as it can happen the file fail to exist right away on eventual consistent fs like Amazon S3 Retryer<Void> retryer = RetryerFactory.newInstance(retrierConfig); try { retryer.call(() -> { if (!fs.exists(path)) { throw new IOException("Path " + path + " does not exist however it should. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException("Path " + path + "does not exist however it should. Giving up..."+ e); } } // Double check permission, since fs.mkdirs() may not guarantee to set the permission correctly if (!fs.getFileStatus(path).getPermission().equals(perm)) { fs.setPermission(path, perm); } }
private void moveTmpPathToOutputPath() throws IOException { Retryer<Void> retryer = RetryerFactory.newInstance(this.retrierConfig); LOG.info(String.format("Moving %s to %s", this.dataset.outputTmpPath(), this.dataset.outputPath())); this.fs.delete(this.dataset.outputPath(), true); if (this.isRetryEnabled) { try { retryer.call(() -> { if (fs.exists(this.dataset.outputPath())) { throw new IOException("Path " + this.dataset.outputPath() + " exists however it should not. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException(e); } } WriterUtils.mkdirsWithRecursivePermissionWithRetry(MRCompactorJobRunner.this.fs, this.dataset.outputPath().getParent(), this.perm, this.retrierConfig); Log.info("Moving from fs: ("+MRCompactorJobRunner.this.tmpFs.getUri()+") path: "+ this.dataset.outputTmpPath() + " to "+ "fs: ("+ FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()).getUri()+") output path: " + this.dataset.outputPath()); HadoopUtils.movePath (MRCompactorJobRunner.this.tmpFs, this.dataset.outputTmpPath(), FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), this.dataset.outputPath(), false, this.fs.getConf()) ; }
.withFallback(POLL_RETRY_DEFAULTS); Retryer<UnsampledReport> retryer = RetryerFactory.newInstance(config); LOG.info("Will poll for completion on unsampled report with retry config: " + config);
Retryer<UnsampledReport> retryer = RetryerFactory.newInstance(config);
public static void mkdirsWithRecursivePermissionWithRetry(final FileSystem fs, final Path path, FsPermission perm, Config retrierConfig) throws IOException { if (fs.exists(path)) { return; } if (path.getParent() != null && !fs.exists(path.getParent())) { mkdirsWithRecursivePermissionWithRetry(fs, path.getParent(), perm, retrierConfig); } if (!fs.mkdirs(path, perm)) { throw new IOException(String.format("Unable to mkdir %s with permission %s", path, perm)); } if (retrierConfig != NO_RETRY_CONFIG) { //Wait until file is not there as it can happen the file fail to exist right away on eventual consistent fs like Amazon S3 Retryer<Void> retryer = RetryerFactory.newInstance(retrierConfig); try { retryer.call(() -> { if (!fs.exists(path)) { throw new IOException("Path " + path + " does not exist however it should. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException("Path " + path + "does not exist however it should. Giving up..."+ e); } } // Double check permission, since fs.mkdirs() may not guarantee to set the permission correctly if (!fs.getFileStatus(path).getPermission().equals(perm)) { fs.setPermission(path, perm); } }
private void moveTmpPathToOutputPath() throws IOException { Retryer<Void> retryer = RetryerFactory.newInstance(this.retrierConfig); LOG.info(String.format("Moving %s to %s", this.dataset.outputTmpPath(), this.dataset.outputPath())); this.fs.delete(this.dataset.outputPath(), true); if (this.isRetryEnabled) { try { retryer.call(() -> { if (fs.exists(this.dataset.outputPath())) { throw new IOException("Path " + this.dataset.outputPath() + " exists however it should not. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException(e); } } WriterUtils.mkdirsWithRecursivePermissionWithRetry(MRCompactorJobRunner.this.fs, this.dataset.outputPath().getParent(), this.perm, this.retrierConfig); Log.info("Moving from fs: ("+MRCompactorJobRunner.this.tmpFs.getUri()+") path: "+ this.dataset.outputTmpPath() + " to "+ "fs: ("+ FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()).getUri()+") output path: " + this.dataset.outputPath()); HadoopUtils.movePath (MRCompactorJobRunner.this.tmpFs, this.dataset.outputTmpPath(), FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), this.dataset.outputPath(), false, this.fs.getConf()) ; }
.withFallback(POLL_RETRY_DEFAULTS); Retryer<UnsampledReport> retryer = RetryerFactory.newInstance(config); LOG.info("Will poll for completion on unsampled report with retry config: " + config);
Retryer<UnsampledReport> retryer = RetryerFactory.newInstance(config);