private static void checkDir(final FileSystem fs, final Path dir) throws IOException { if (fs.exists(dir)) { throw new RuntimeException("The " + dir + " exists"); } if (!fs.mkdirs(dir)) { throw new IOException("Failed to create the " + dir); } }
private static void moveFiles(final FileSystem fs, final Path src, final Path dest) throws IOException { try { // create the dest directory if not exist if (!fs.exists(dest.getParent())) { fs.mkdirs(dest.getParent()); } // if the destination file exists for some reason delete it fs.delete(dest, false); if (fs.rename(src, dest)) { System.err.println("Moved " + src + " to " + dest); } else { throw new IOException("Unable to move " + src + " to " + dest); } } catch (Exception e) { throw new IOException("Unable to move " + src + " to " + dest, e); } }
private static List<Path> copyFiles(FileSystem srcFS, Path src, FileSystem dstFS, Path dst, Configuration conf, ExecutorService pool, List<Future<Void>> futures) throws IOException { List<Path> traversedPaths = new ArrayList<>(); traversedPaths.add(dst); FileStatus currentFileStatus = srcFS.getFileStatus(src); if (currentFileStatus.isDirectory()) { if (!dstFS.mkdirs(dst)) { throw new IOException("create dir failed: " + dst); } FileStatus[] subPaths = srcFS.listStatus(src); for (FileStatus subPath : subPaths) { traversedPaths.addAll(copyFiles(srcFS, subPath.getPath(), dstFS, new Path(dst, subPath.getPath().getName()), conf, pool, futures)); } } else { Future<Void> future = pool.submit(() -> { FileUtil.copy(srcFS, src, dstFS, dst, false, false, conf); return null; }); futures.add(future); } return traversedPaths; } }
@Override public void put(String jobName, String datasetUrn, CommitSequence commitSequence) throws IOException { datasetUrn = sanitizeDatasetUrn(datasetUrn); if (exists(jobName, datasetUrn)) { throw new IOException(String.format("CommitSequence already exists for job %s, dataset %s", jobName, datasetUrn)); } Path jobPath = new Path(this.rootPath, jobName); this.fs.mkdirs(jobPath); Path datasetPath = new Path(jobPath, datasetUrn); try (DataOutputStream dos = this.fs.create(datasetPath)) { dos.writeBytes(GSON.toJson(commitSequence)); } }
public static void createDirectory(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) { try { if (!hdfsEnvironment.getFileSystem(context, path).mkdirs(path, ALL_PERMISSIONS)) { throw new IOException("mkdirs returned false"); } } catch (IOException e) { throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed to create directory: " + path, e); } // explicitly set permission since the default umask overrides it on creation try { hdfsEnvironment.getFileSystem(context, path).setPermission(path, ALL_PERMISSIONS); } catch (IOException e) { throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed to set permission on directory: " + path, e); } }
@After public void cleanUp() throws IOException { // delete and recreate the test directory, ensuring a clean test dir between tests Path testDir = UTIL.getDataTestDir(); FileSystem fs = UTIL.getTestFileSystem(); fs.delete(testDir, true); if (!fs.mkdirs(testDir)) throw new IOException("Failed mkdir " + testDir); }
public static void moveSelectFiles(String extension, String source, String destination) throws IOException { FileSystem fs = getFileSystem(); fs.mkdirs(new Path(destination)); FileStatus[] fileStatuses = fs.listStatus(new Path(source)); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); if (!fileStatus.isDirectory() && path.toString().toLowerCase().endsWith(extension.toLowerCase())) { HadoopUtils.deleteIfExists(fs, new Path(destination), true); HadoopUtils.copyPath(fs, path, fs, new Path(destination), getConfiguration()); } } } }
@BeforeClass public void setup() throws Exception { fs = FileSystem.getLocal(new Configuration()); testClassTempPath = new Path(Files.createTempDir().getAbsolutePath(), "CopyDataPublisherTest"); fs.delete(testClassTempPath, true); log.info("Created a temp directory for CopyDataPublisherTest at " + testClassTempPath); fs.mkdirs(testClassTempPath); }
@Override public Void call() throws Exception { synchronized (fs) { String tezLibs = conf.get(TezConfiguration.TEZ_LIB_URIS); if (tezLibs == null) { LOG.warn("Missing tez.lib.uris in tez-site.xml"); } if (LOG.isDebugEnabled()) { LOG.debug("Copying tez libs from " + tezLibs); } lfs.mkdirs(tezDir); fs.copyToLocalFile(new Path(tezLibs), new Path(libDir, "tez.tar.gz")); CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), tezDir.toString(), true); lfs.delete(new Path(libDir, "tez.tar.gz"), false); } return null; } };
/** @throws Exception If failed. */ @Test public void testSetWorkingDirectory() throws Exception { Path dir = new Path("/tmp/nested/dir"); Path file = new Path("file"); fs.mkdirs(dir); fs.setWorkingDirectory(dir); FSDataOutputStream os = fs.create(file); os.close(); String filePath = fs.getFileStatus(new Path(dir, file)).getPath().toString(); assertTrue(filePath.contains("/tmp/nested/dir/file")); }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.delete(new Path(ROOT_DIR), true); Path dir1 = new Path(ROOT_DIR, "dir1"); Path dir2 = new Path(ROOT_DIR, "dir2"); this.fs.mkdirs(dir1); this.fs.mkdirs(dir2); Path src = new Path(dir1, "file"); Path dst = new Path(dir2, "file"); this.fs.createNewFile(src); this.step = (FsRenameCommitStep) new FsRenameCommitStep.Builder<>().from(src).to(dst).withProps(new State()).build(); }
private void createDirIfNotExists(Path path) throws IOException { FileSystem fileSystem = path.getFileSystem(conf); try { if (!fileSystem.exists(path)) { fileSystem.mkdirs(path); fileSystem.setPermission(path, DIR_PERMISSION); } } catch (IOException e) { // Ignore this exception, if there is a problem it'll fail when trying to read or write. LOG.warn("Error while trying to set permission: ", e); } }
private <T> void writeFile(String type, Path path, JsonCodec<T> codec, T value, boolean overwrite) { try { byte[] json = codec.toJsonBytes(value); if (!overwrite) { if (metadataFileSystem.exists(path)) { throw new PrestoException(HIVE_METASTORE_ERROR, type + " file already exists"); } } metadataFileSystem.mkdirs(path.getParent()); // todo implement safer overwrite code try (OutputStream outputStream = metadataFileSystem.create(path, overwrite)) { outputStream.write(json); } } catch (Exception e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not write " + type, e); } }
@Override public UploadHandle initialize(Path filePath) throws IOException { Path collectorPath = createCollectorPath(filePath); fs.mkdirs(collectorPath, FsPermission.getDirDefault()); ByteBuffer byteBuffer = ByteBuffer.wrap( collectorPath.toString().getBytes(Charsets.UTF_8)); return BBUploadHandle.from(byteBuffer); }
/** * Safe creation of trash folder to ensure thread-safe. * @throws IOException */ private boolean safeFsMkdir(FileSystem fs, Path f, FsPermission permission) throws IOException { try { return fs.mkdirs(f, permission); } catch (IOException e) { // To handle the case when trash folder is created by other threads // The case is rare and we don't put synchronized keywords for performance consideration. if (!fs.exists(f)) { throw new IOException("Failed to create trash folder while it is still not existed yet."); } else { LOG.debug("Target folder %s has been created by other threads.", f.toString()); return true; } } } }
/** * createTezDir creates a temporary directory in the scratchDir folder to * be used with Tez. Assumes scratchDir exists. */ public Path createTezDir(Path scratchDir, Configuration conf) throws IOException { UserGroupInformation ugi; String userName = System.getProperty("user.name"); try { ugi = Utils.getUGI(); userName = ugi.getShortUserName(); } catch (LoginException e) { throw new IOException(e); } scratchDir = new Path(scratchDir, userName); Path tezDir = getTezDir(scratchDir); FileSystem fs = tezDir.getFileSystem(conf); LOG.debug("TezDir path set " + tezDir + " for user: " + userName); // since we are adding the user name to the scratch dir, we do not // need to give more permissions here fs.mkdirs(tezDir); return tezDir; }