@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
private void createDirectory(FileContext fileContext, String path) { try { org.apache.hadoop.fs.Path fPath = new org.apache.hadoop.fs.Path(path); boolean dirExists = checkDirectoryExists(fileContext, fPath); if (!dirExists) { FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL); // file context does ( permission AND (NOT of umask) ) and uses that as permission, by default umask is 022, // if we want 777 permission, we have to set umask to 000 fileContext.setUMask(new FsPermission(FsAction.NONE, FsAction.NONE, FsAction.NONE)); fileContext.mkdir(fPath, permission, true); } } catch (FileAlreadyExistsException e) { // should not happen as we create only if dir exists } catch (AccessControlException | ParentNotDirectoryException | FileNotFoundException e) { // just log the exception LOG.error("Exception while trying to create directory at {}", path, e); } catch (IOException e) { throw Throwables.propagate(e); } }
new Path(doneDirPrefix)); doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf); doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK); mkdir(doneDirFc, doneDirPrefixPath, new FsPermission( JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
new Path(doneDirPrefix)); doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf); doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK); mkdir(doneDirFc, doneDirPrefixPath, new FsPermission( JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
fc = FileContext.getFileContext( remoteRootLogDir.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); if (indexedLogsMeta == null) { indexedLogsMeta = new IndexedLogsMeta();
@Before public void setUp() throws Exception { fc.setUMask(WIDE_OPEN_TEST_UMASK); fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true); }
lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));
@Override public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { List<String> localDirs = dirsHandler.getLocalDirs(); List<String> logDirs = dirsHandler.getLogDirs(); createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); createAppLogDirs(appId, logDirs, user); // randomly choose the local directory Path appStorageDir = getWorkingDir(localDirs, user, appId); String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); Path tokenDst = new Path(appStorageDir, tokenFn); copyFile(nmPrivateContainerTokensPath, tokenDst, user); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); FileContext localizerFc = FileContext.getFileContext( lfs.getDefaultFileSystem(), getConf()); localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir); LOG.info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.getWorkingDirectory()); ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); }
lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));
@Override public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { List<String> localDirs = dirsHandler.getLocalDirs(); List<String> logDirs = dirsHandler.getLogDirs(); createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); createAppLogDirs(appId, logDirs, user); // randomly choose the local directory Path appStorageDir = getWorkingDir(localDirs, user, appId); String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); Path tokenDst = new Path(appStorageDir, tokenFn); copyFile(nmPrivateContainerTokensPath, tokenDst, user); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); FileContext localizerFc = FileContext.getFileContext( lfs.getDefaultFileSystem(), getConf()); localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir); LOG.info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.getWorkingDirectory()); ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); }
localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir); LOG.info("Localizer CWD set to " + appStorageDir + " = "
lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));
lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));
localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir); LOG.info("Localizer CWD set to " + appStorageDir + " = "
public void testCreateRecursiveWithExistingDir(FsPermission umask, FsPermission expectedPerms) throws IOException { Path f = fileContextTestHelper.getTestRootPath(fc,"foo"); fc.setUMask(umask); createFile(fc, f); Assert.assertTrue(isFile(fc, f)); Assert.assertEquals("permissions on file are wrong", expectedPerms , fc.getFileStatus(f).getPermission()); }
public void testMkdirWithExistingDir(FsPermission umask, FsPermission expectedPerms) throws IOException { Path f = fileContextTestHelper.getTestRootPath(fc, "aDir"); fc.setUMask(umask); fc.mkdir(f, FileContext.DEFAULT_PERM, true); Assert.assertTrue(isDir(fc, f)); Assert.assertEquals("permissions on directory are wrong", expectedPerms, fc.getFileStatus(f).getPermission()); }
public void testMkdirRecursiveWithNonExistingDir(FsPermission umask, FsPermission expectedPerms, FsPermission expectedParentPerms) throws IOException { Path f = fileContextTestHelper.getTestRootPath(fc, "NonExistant2/aDir"); fc.setUMask(umask); fc.mkdir(f, FileContext.DEFAULT_PERM, true); Assert.assertTrue(isDir(fc, f)); Assert.assertEquals("permissions on directory are wrong", expectedPerms, fc.getFileStatus(f).getPermission()); Path fParent = fileContextTestHelper.getTestRootPath(fc, "NonExistant2"); Assert.assertEquals("permissions on parent directory are wrong", expectedParentPerms, fc.getFileStatus(fParent).getPermission()); }
public void testCreateRecursiveWithNonExistingDir(FsPermission umask, FsPermission expectedDirPerms, FsPermission expectedFilePerms) throws IOException { Path f = fileContextTestHelper.getTestRootPath(fc,"NonExisting/foo"); Path fParent = fileContextTestHelper.getTestRootPath(fc, "NonExisting"); Assert.assertFalse(exists(fc, fParent)); fc.setUMask(umask); createFile(fc, f); Assert.assertTrue(isFile(fc, f)); Assert.assertEquals("permissions on file are wrong", expectedFilePerms, fc.getFileStatus(f).getPermission()); Assert.assertEquals("permissions on parent directory are wrong", expectedDirPerms, fc.getFileStatus(fParent).getPermission()); }