Refine search
/** * Returns all jars that are in the directory. It is useful in expanding a * wildcard path to return all jars from the directory to use in a classpath. * * @param path the path to the directory. The path may include the wildcard. * @return the list of jars as URLs, or an empty list if there are no jars, or * the directory does not exist */ public static List<Path> getJarsInDirectory(String path, boolean useLocal) { List<Path> paths = new ArrayList<>(); try { // add the wildcard if it is not provided if (!path.endsWith("*")) { path += File.separator + "*"; } Path globPath = new Path(path).suffix("{.jar,.JAR}"); FileContext context = useLocal ? FileContext.getLocalFSFileContext() : FileContext.getFileContext(globPath.toUri()); FileStatus[] files = context.util().globStatus(globPath); if (files != null) { for (FileStatus file: files) { paths.add(file.getPath()); } } } catch (IOException ignore) {} // return the empty list return paths; }
ParentNotDirectoryException, UnsupportedFileSystemException, IOException { src.checkNotSchemeWithRelative(); dst.checkNotSchemeWithRelative(); Path qSrc = makeQualified(src); Path qDst = makeQualified(dst); checkDest(qSrc.getName(), qDst, overwrite); FileStatus fs = FileContext.this.getFileStatus(qSrc); if (fs.isDirectory()) { checkDependencies(qSrc, qDst); mkdir(qDst, FsPermission.getDirDefault(), true); FileStatus[] contents = listStatus(qSrc); for (FileStatus content : contents) { copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, content.getPath().getName())), deleteSource, overwrite); EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of( CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE); InputStream in = open(qSrc); try (OutputStream out = create(qDst, createFlag)) { IOUtils.copyBytes(in, out, conf, true); } finally { return delete(qSrc, true); } else { return true;
@Override public void commit() throws IOException { checkIsNotTmp(); // FileContext supports atomic rename, whereas FileSystem doesn't FileContext fc = FileContext.getFileContext(_hadoopConf); Path dest = new Path(_path.getParent(), BLOBSTORE_DATA_FILE); if (_mustBeNew) { fc.rename(_path, dest); } else { fc.rename(_path, dest, Options.Rename.OVERWRITE); } // Note, we could add support for setting the replication factor }
/** Construct given a {@link FileContext} and a {@link Path}. */ public AvroFSInput(final FileContext fc, final Path p) throws IOException { FileStatus status = fc.getFileStatus(p); this.len = status.getLen(); this.stream = fc.open(p); }
/** * Create a FileContext for specified default URI using the specified config. * * @param defaultFsUri * @param aConf * @return new FileContext for specified uri * @throws UnsupportedFileSystemException If the file system with specified is * not supported * @throws RuntimeException If the file system specified is supported but * could not be instantiated, or if login fails. */ public static FileContext getFileContext(final URI defaultFsUri, final Configuration aConf) throws UnsupportedFileSystemException { UserGroupInformation currentUser = null; AbstractFileSystem defaultAfs = null; if (defaultFsUri.getScheme() == null) { return getFileContext(aConf); } try { currentUser = UserGroupInformation.getCurrentUser(); defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf); } catch (UnsupportedFileSystemException ex) { throw ex; } catch (IOException ex) { LOG.error(ex.toString()); throw new RuntimeException(ex); } return getFileContext(defaultAfs, aConf); }
public void checkFileStatus(String path, fileType expectedType) throws IOException { FileStatus s = fc.getFileStatus(new Path(path)); Assert.assertNotNull(s); if (expectedType == fileType.isDir) { Assert.assertTrue(s.isDirectory()); } else if (expectedType == fileType.isFile) { Assert.assertTrue(s.isFile()); } else if (expectedType == fileType.isSymlink) { Assert.assertTrue(s.isSymlink()); } Assert.assertEquals(fc.makeQualified(new Path(path)), s.getPath()); }
File targetWorkDir = new File("target", clusterNameTrimmed); try { FileContext.getLocalFSFileContext().delete( new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { LOG.warn("Could not cleanup test workDir: " + targetWorkDir, e);
@BeforeClass public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); fc = FileContext.getFileContext(cluster.getURI(), conf); defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); }
@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
@Test(expected=AccessControlException.class) public void testInternalDeleteExisting2() throws IOException { Assert.assertTrue("Delete of link to dir should succeed", fcView.getFileStatus(new Path("/internalDir/linkToDir2")).isDirectory()); fcView.delete(new Path("/internalDir/linkToDir2"), false); }
/** The list operation randomly picks a directory in the test space and * list the directory content. */ private void list() throws IOException { String dirName = dirs.get(r.nextInt(dirs.size())); long startTime = Time.now(); fc.listStatus(new Path(dirName)); executionTime[LIST] += (Time.now()-startTime); totalNumOfOps[LIST]++; }
@Test(expected=AccessControlException.class) public void testInternalRename2() throws IOException { Assert.assertTrue("linkTODir2 should be a dir", fcView.getFileStatus(new Path("/internalDir/linkToDir2")).isDirectory()); fcView.rename(new Path("/internalDir/linkToDir2"), new Path("/internalDir/dir1")); } @Test(expected=AccessControlException.class)
/** * Override this so that we don't set the targetTestRoot to any path under the * root of the FS, and so that we don't try to delete the test dir, but rather * only its contents. */ @Override void initializeTargetTestRoot() throws IOException { targetTestRoot = fc.makeQualified(new Path("/")); RemoteIterator<FileStatus> dirContents = fc.listStatus(targetTestRoot); while (dirContents.hasNext()) { fc.delete(dirContents.next().getPath(), true); } }
CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(FileContext.getFileContext(fs.getUri(), conf), conf, name, keyClass, valClass, compressionType, codec, metadata, EnumSet.of(CreateFlag.CREATE,CreateFlag.OVERWRITE), CreateOpts.bufferSize(bufferSize), createParent ? CreateOpts.createParent()
/** * Check if copying srcName to dst would overwrite an existing * file or directory. * @param srcName File or directory to be copied. * @param dst Destination to copy srcName to. * @param overwrite Whether it's ok to overwrite an existing file. * @throws AccessControlException If access is denied. * @throws IOException If dst is an existing directory, or dst is an * existing file and the overwrite option is not passed. */ private void checkDest(String srcName, Path dst, boolean overwrite) throws AccessControlException, IOException { try { FileStatus dstFs = getFileStatus(dst); if (dstFs.isDirectory()) { if (null == srcName) { throw new IOException("Target " + dst + " is a directory"); } // Recurse to check if dst/srcName exists. checkDest(null, new Path(dst, srcName), overwrite); } else if (!overwrite) { throw new IOException("Target " + new Path(dst, srcName) + " already exists"); } } catch (FileNotFoundException e) { // dst does not exist - OK to copy. } }
/** * We would have liked renames across file system to fail but * Unfortunately there is not way to distinguish the two file systems * @throws IOException */ @Test public void testRenameAcrossFs() throws IOException { fc.mkdir(new Path("/newDir/dirFoo"), FileContext.DEFAULT_PERM, true); // the root will get interpreted to the root of the chrooted fs. fc.rename(new Path("/newDir/dirFoo"), new Path("file:///dirFooBar")); FileContextTestHelper.isDir(fc, new Path("/dirFooBar")); }
private void findFiles(FSWindowReplayWAL wal, int operatorId) throws IOException { String operatorDir = fullStatePath + Path.SEPARATOR + operatorId; Path operatorPath = new Path(operatorDir); if (fileContext.util().exists(operatorPath)) { RemoteIterator<FileStatus> walFilesIter = fileContext.listStatus(operatorPath); while (walFilesIter.hasNext()) { FileStatus fileStatus = walFilesIter.next(); FSWindowReplayWAL.FileDescriptor descriptor = FSWindowReplayWAL.FileDescriptor.create(fileStatus.getPath()); wal.fileDescriptors.put(descriptor.part, descriptor); } } }
this.fileContext = FileContext.getFileContext(conf); .getWriterStagingDir(properties, this.numBranches, this.branchId, this.writerAttemptIdOptional.get()) : WriterUtils.getWriterStagingDir(properties, this.numBranches, this.branchId); this.stagingFile = new Path(writerStagingDir, this.fileName); new Path(WriterUtils.getWriterOutputDir(properties, this.numBranches, this.branchId), this.fileName); this.allOutputFilesPropName = ForkOperatorUtils .getPropertyNameForBranch(ConfigurationKeys.WRITER_FINAL_OUTPUT_FILE_PATHS, this.numBranches, this.branchId); WriterUtils.mkdirsWithRecursivePermission(this.fs, this.outputFile.getParent(), this.dirPermission); this.bytesWritten = Optional.absent();