/** * validate if a file on HDFS can be open * * @throws IOException the file can't be found or opened * @throws URISyntaxException */ public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException { try(InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(new URI(pathStr)))) { if (inputStream == null) { throw new IOException("HDFS file " + pathStr + " can't be opened"); } } }
/** * Get the filesystem of this cluster * @return the FS of the config */ public SliderFileSystem getClusterFS() throws IOException { return new SliderFileSystem(getConfig()); }
private int actionDeleteResource(ActionResourceArgs resourceInfo) throws BadCommandArgumentsException, IOException { if (StringUtils.isEmpty(resourceInfo.resource)) { throw new BadCommandArgumentsException("A file name is required."); } Path fileInFs; if (resourceInfo.folder == null) { fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource); } else { fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder, resourceInfo.resource); } log.info("Deleting resource {}", fileInFs); FileSystem sfs = sliderFileSystem.getFileSystem(); require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri()); sfs.delete(fileInFs, true); return EXIT_SUCCESS; }
templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE)); if (!fileSystem.isFile(templateFile)) { templateFile = fileSystem.buildResourcePath(clusterName, config.get(TEMPLATE_FILE)); if (!fileSystem.isFile(templateFile)) { throw new IOException("config specified template file " + config .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist"); templateFile = fileSystem.buildResourcePath(fileName); if (!fileSystem.isFile(templateFile)) { templateFile = fileSystem.buildResourcePath(clusterName, fileName); if (fileSystem.isFile(templateFile)) { config.put("content", fileSystem.cat(templateFile)); } else { config.put("content", "");
private void uploadServerCertForLocalization(String clustername, SliderFileSystem fs) throws IOException { Path certsDir = fs.buildClusterSecurityDirPath(clustername); if (!fs.getFileSystem().exists(certsDir)) { fs.getFileSystem().mkdirs(certsDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); } Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME); if (!fs.getFileSystem().exists(destPath)) { fs.getFileSystem().copyFromLocalFile( new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()), destPath); log.info("Uploaded server cert to localization path {}", destPath); } fs.getFileSystem().setPermission(destPath, new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); }
if (fileSystem.getFileSystem().exists(agentImagePath)) { LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes); } else { LocalResource appDefRes = fileSystem.createAmResource( fileSystem.getFileSystem().resolvePath(new Path(appDef)), LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes); Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); if (!fileSystem.isFile(pkgPath)) { pkgPath = fileSystem.buildResourcePath(getClusterName(), pkg.getName()); if (!fileSystem.isFile(pkgPath)) { throw new IOException("Package doesn't exist as a resource: " + pkg.getName()); LocalResource packageResource = fileSystem.createAmResource( pkgPath, type); launcher.addLocalResource(AgentKeys.APP_PACKAGES_DIR, packageResource); getGlobalOptions().getOption(AgentKeys.AGENT_CONF, ""); if (SliderUtils.isSet(agentConf)) { LocalResource agentConfRes = fileSystem.createAmResource(fileSystem .getFileSystem().resolvePath(new Path(agentConf)), LocalResourceType.FILE); launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
UserGroupInformation.getCurrentUser().getCredentials()); CredentialUtils.addRMRenewableFSDelegationTokens(config, sliderFileSystem.getFileSystem(), credentials); CredentialUtils.addRMDelegationToken(yarnClient, credentials); sliderFileSystem.purgeAppInstanceTempFiles(clustername); Path tempPath = sliderFileSystem.createAppInstanceTempPath( clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); sliderFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath); relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); mergeMaps(localResources, submittedConfDir); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); log.debug("Environment Map:\n{}", stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); FileSystem mockFs = new MockFileSystem(); expect(sliderFileSystem.getFileSystem()).andReturn( new FilterFileSystem(mockFs)).anyTimes(); expect( sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class))).andReturn( createNiceMock(LocalResource.class)).anyTimes();
SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, getClusterName()); boolean serviceKeytabsDeployed = false; if (fileSystem.getFileSystem().exists(keytabDirPath)) { FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath); LocalResource keytabRes; for (FileStatus keytab : keytabs) { serviceKeytabsDeployed = true; log.info("Localizing keytab {}", keytab.getPath().getName()); keytabRes = fileSystem.createAmResource(keytab.getPath(), LocalResourceType.FILE); launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
@Test public void testGetMetaInfoStreamFromZip() throws Exception { String zipFileName = TestUtility.createAppPackage( folder, "testpkg", "test.zip", "target/test-classes/org/apache/slider/common/tools/test"); Configuration configuration = new Configuration(); FileSystem fs = FileSystem.getLocal(configuration); log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); InputStream stream = SliderUtils.getApplicationResourceInputStream( sliderFileSystem.getFileSystem(), new Path(zipFileName), "metainfo.xml"); Assert.assertTrue(stream != null); Assert.assertTrue(stream.available() > 0); }
private int actionListKeytab(ActionKeytabArgs keytabInfo) throws IOException { String folder = keytabInfo.folder != null ? keytabInfo.folder : StringUtils.EMPTY; Path keytabPath = sliderFileSystem.buildKeytabInstallationDirPath(folder); RemoteIterator<LocatedFileStatus> files = sliderFileSystem.getFileSystem().listFiles(keytabPath, true); log.info("Keytabs:"); while (files.hasNext()) { log.info("\t" + files.next().getPath().toString()); } return EXIT_SUCCESS; }
private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws YarnException, IOException { requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, actionPackageArgs.version); FileSystem fs = sliderFileSystem.getFileSystem(); require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri()); log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath); if(fs.delete(pkgPath, true)) { log.info("Deleted package {} " + actionPackageArgs.name); return EXIT_SUCCESS; } else { log.warn("Package deletion failed."); return EXIT_NOT_FOUND; } }
launcher.addLocalResources(coreFS.submitDirectory( generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME)); coreFS.maybeAddImagePath(launcher.getLocalResources(), imageURI);
Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); if (!fileSystem.isFile(pkgPath) && name != null) { pkgPath = fileSystem.buildResourcePath(name, pkg.getName()); if (!fileSystem.isFile(pkgPath)) { throw new IOException("Package doesn't exist as a resource: " + pkg.getName()); fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile); expandTar(pkgFile, clientInstallPath); } else { File pkgFile = new File(clientInstallPath, pkg.getName()); fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
log.info("Loading all dependencies for AM."); Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); if (fileSystem.isFile(dependencyLibTarGzip)) { SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); } else {
/** * Create the cluster -saving the arguments to a specification file first * @param clustername cluster name * @return the status code * @throws YarnException Yarn problems * @throws IOException other problems * @throws BadCommandArgumentsException bad arguments. */ public int actionCreate(String clustername, ActionCreateArgs createArgs) throws YarnException, IOException { actionBuild(clustername, createArgs); Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clustername, clusterDirectory); try { checkForCredentials(getConfig(), instanceDefinition.getAppConf(), clustername); } catch (IOException e) { sliderFileSystem.getFileSystem().delete(clusterDirectory, true); throw e; } return startCluster(clustername, createArgs); }
Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip(); if (sliderFileSystem.isFile(dependencyLibTarGzip) && !args.overwrite) { println(String.format( "Dependency libs are already uploaded to %s. Use %s " sliderFileSystem.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip, new FsPermission( SliderKeys.SLIDER_DEPENDENCY_DIR_PERMISSIONS));
pkgFile = new File(clientInfo.packageURI); } else { Path appDirPath = sliderFileSystem.buildAppDefDirPath(clientInfo.name); Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); require(sliderFileSystem.isFile(appDefPath), E_INVALID_APPLICATION_PACKAGE_LOCATION); tmpDir = Files.createTempDir(); pkgFile = new File(tmpDir, SliderKeys.DEFAULT_APP_PKG); sliderFileSystem.copyHdfsFileToLocal(appDefPath, pkgFile);
try { Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName()); LocalResource certResource = fileSystem.createAmResource( new Path(certsDir, SliderKeys.CRT_FILE_NAME), LocalResourceType.FILE); String containerId = container.getId().toString(); certMgr.generateContainerCertificate(hostname, containerId); LocalResource agentCertResource = fileSystem.createAmResource( uploadSecurityResource( CertificateManager.getAgentCertficateFilePath(containerId), LocalResource agentKeyResource = fileSystem.createAmResource( uploadSecurityResource( CertificateManager.getAgentKeyFilePath(containerId), fileSystem),
new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission);