Refine search
conf.addResource(new Path(XLearningConstants.XLEARNING_JOB_CONFIGURATION)); System.setProperty(XLearningConstants.Environment.HADOOP_USER_NAME.toString(), conf.get("hadoop.job.ugi").split(",")[0]); outputInfos = new ArrayList<>(); input2FileStatus = new ConcurrentHashMap<>(); inputFileSplits = null; containerId2InputSplit = new ConcurrentHashMap<>(); statusUpdateInterval = conf.getInt(XLearningConfiguration.XLEARNING_STATUS_UPDATE_INTERVAL, XLearningConfiguration.DEFAULT_XLEARNING_STATUS_PULL_INTERVAL); applicationAttemptID = Records.newRecord(ApplicationAttemptId.class); applicationMessageQueue = new LinkedBlockingQueue<>( .toContainerId(envs.get(ApplicationConstants.Environment.CONTAINER_ID.toString())); applicationAttemptID = containerId.getApplicationAttemptId(); } else { appJarRemoteLocation = new Path(envs.get(XLearningConstants.Environment.APP_JAR_LOCATION.toString())); LOG.info("Application jar location: " + appJarRemoteLocation); appConfRemoteLocation = new Path(envs.get(XLearningConstants.Environment.XLEARNING_JOB_CONF_LOCATION.toString())); LOG.info("Application conf location: " + appConfRemoteLocation);
public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(path); localResource.setResource(ConverterUtils.getYarnUrlFromPath(path)); localResource.setSize(fileStatus.getLen()); localResource.setTimestamp(fileStatus.getModificationTime()); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
@Override @Nullable public ApplicationId getClusterId(CommandLine commandLine) { if (commandLine.hasOption(applicationId.getOpt())) { return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt())); } else if (isYarnPropertiesFileMode(commandLine)) { return yarnApplicationIdFromYarnProperties; } else { return null; } }
if (cliParser.hasOption(JOYConstants.APP_ATTEMPT_ID)) { String appIdStr = cliParser.getOptionValue(JOYConstants.APP_ATTEMPT_ID, JOYConstants.EMPTY); jstormMasterContext.appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException( ContainerId containerId = ConverterUtils.toContainerId(envs .get(ApplicationConstants.Environment.CONTAINER_ID.name())); jstormMasterContext.appAttemptID = containerId.getApplicationAttemptId(); conf.set(JOYConstants.INSTANCE_DEPLOY_DIR_KEY, envs.get(JOYConstants.BINARYFILEDEPLOYPATH)); jstormMasterContext.deployPath = envs.get(JOYConstants.BINARYFILEDEPLOYPATH); conf.set(JOYConstants.INSTANCE_NAME_KEY, envs.get(JOYConstants.INSTANCENAME)); jstormMasterContext.instanceName = envs.get(JOYConstants.INSTANCENAME);
yarnUrl = ConverterUtils.getYarnUrlFromURI( new URI(renamedScriptPath.toString())); } catch (URISyntaxException e) { LOG.error("Error when trying to use shell script path specified" jarUrl = ConverterUtils.getYarnUrlFromURI( new URI(jstormMasterContext.appMasterJarPath.toString())); } catch (URISyntaxException e) { LocalResource shellRsrc = LocalResource.newInstance(yarnUrl, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, jstormMasterContext.shellScriptPathLen, jstormMasterContext.shellScriptPathTimestamp); localResources.put(JOYConstants.ExecShellStringPath, shellRsrc); LocalResource jarRsrc = LocalResource.newInstance(jarUrl, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, jstormMasterContext.jarPathLen, jstormMasterContext.jarTimestamp); localResources.put(JOYConstants.appMasterJarPath, jarRsrc); LOG.info(shellRsrc.getResource().getFile()); jstormMasterContext.nimbusDataDirPrefix = conf.get(JOYConstants.INSTANCE_DATA_DIR_KEY); String localDir = jstormMasterContext.nimbusDataDirPrefix + container.getId().toString() + JOYConstants.BACKLASH + jstormMasterContext.instanceName;
assertFalse(targetFileSystem.exists(targetDir)); srcPath = new Path("file://" + srcDir.getAbsolutePath()); } else { srcPath = new Path(srcDir.getAbsolutePath()); HashMap<String, LocalResource> localResources = new HashMap<>(); AbstractYarnClusterDescriptor.uploadAndRegisterFiles( Collections.singletonList(new File(srcPath.toUri().getPath())), targetFileSystem, targetDir, .getPathFromYarnURL(localResources.get(srcPath.getName() + "/1").getResource()) .getParent(); targetFileSystem.listFiles(workDir, true); HashMap<String /* (relative) path */, /* contents */ String> targetFiles = new HashMap<>(4); try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { String absolutePathString = targetFile.getPath().toString(); String relativePath = absolutePathString.substring(workDirPrefixLength);
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
public void testCreateLocalResourceForConfigurationFile() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); String filename = "foo.xml"; Configuration conf = has.createConfiguration(getNameNodeUri()); conf.set("foo", "bar"); LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(), getFsTestCaseDir()); assertNotNull(lRes); assertEquals(LocalResourceType.FILE, lRes.getType()); assertEquals(LocalResourceVisibility.APPLICATION, lRes.getVisibility()); Path resPath = ConverterUtils.getPathFromYarnURL(lRes.getResource()); assertEquals(new Path(getFsTestCaseDir(), "foo.xml"), resPath); Configuration conf2 = new Configuration(false); conf2.addResource(getFileSystem().open(resPath)); assertEquals("bar", conf2.get("foo")); } }
Path dir) throws IOException, HadoopAccessorException, URISyntaxException { Path dst = new Path(dir, filename); FileSystem fs = createFileSystem(user, uri, conf, false); try (OutputStream os = fs.create(dst)){ conf.writeXml(os); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); localResource.setResource(ConverterUtils.getYarnUrlFromPath(dst)); FileStatus destStatus = fs.getFileStatus(dst); localResource.setTimestamp(destStatus.getModificationTime()); localResource.setSize(destStatus.getLen()); return localResource;
final Path sCopy; try { sCopy = ConverterUtils.getPathFromYarnURL(resource.getResource()); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); final Path dst_work = new Path(destDirPath + "_tmp"); createDir(dst_work, cachePerms); Path dFinal = files.makeQualified(new Path(dst_work, sCopy.getName())); try { Path dTmp = null == userUgi ? files.makeQualified(copy(sCopy, dst_work))
private List<LocalResource> getDataNodeBlockListingFiles() throws IOException { Path blockListDirPath = new Path(System.getenv().get(DynoConstants.BLOCK_LIST_PATH_ENV)); LOG.info("Looking for block listing files in " + blockListDirPath); FileSystem blockZipFS = blockListDirPath.getFileSystem(conf); List<LocalResource> files = new LinkedList<>(); for (FileStatus stat : blockZipFS.listStatus(blockListDirPath, DynoConstants.BLOCK_LIST_FILE_FILTER)) { LocalResource blockListResource = LocalResource.newInstance( ConverterUtils.getYarnUrlFromPath(stat.getPath()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, stat.getLen(), stat.getModificationTime()); files.add(blockListResource); } return files; }
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName, Map<String, LocalResource> localResources) throws IOException { Path dstPath = new Path(fileDstPath, fileName); dstPath = fs.makeQualified(dstPath); fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath); FileStatus fileStatus = fs.getFileStatus(dstPath); LocalResource localRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dstPath.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime()); localResources.put(fileName, localRsrc); } }
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException { FileSystem fs = path.getFileSystem(clusterConf.conf()); FileStatus stat = fs.getFileStatus(path); return LocalResource.newInstance( ConverterUtils.getYarnUrlFromPath(path), LocalResourceType.FILE, visibility, stat.getLen(), stat.getModificationTime() ); }
private Map.Entry<String, LocalResource> localizeResource(FileStatus status) { URL url = ConverterUtils.getYarnUrlFromURI( status.getPath().toUri()); LocalResource resource = LocalResource.newInstance(url, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime()); String name = status.getPath().getName(); return new AbstractMap.SimpleEntry<>(name, resource); }
boolean verifyAccess() throws IOException { if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { return true; remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource()); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); FileSystem remoteFs = remotePath.getFileSystem(conf); FileStatus status = remoteFs.getFileStatus(remotePath); if (status.getModificationTime() != resource.getTimestamp()) { LOG.warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload"); if (status.getOwner().equals(user)) { return true; // the user owns the file
@Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); URL url = ConverterUtils.getYarnUrlFromPath(expectedPath); Path actualPath = ConverterUtils.getPathFromYarnURL(url); assertEquals(expectedPath, actualPath); }
URL destUrl; try { destUrl = ConverterUtils.getYarnUrlFromPath( FileContext.getFileContext().makeQualified(dfsPath)); } catch (UnsupportedFileSystemException e) { throw new DfsFacadeException( "Unable to convert dfs file to a URL: " + dfsPath.toString(), e); LocalResource resource = LocalResource.newInstance(destUrl, type, visibility, dfsFileStatus.getLen(), dfsFileStatus.getModificationTime()); return resource;
/** * Creates a LocalResource instance for the JAR file referenced by the given Path. */ public LocalResource getLocalResourceForPath(final Path jarPath, final LocalResourceType type) throws IOException { final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath); final LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath())); localResource.setTimestamp(status.getModificationTime()); localResource.setSize(status.getLen()); return localResource; }
private XLearningContainer() { this.conf = new XLearningConfiguration(); conf.addResource(new Path(XLearningConstants.XLEARNING_JOB_CONFIGURATION)); LOG.info("user is " + conf.get("hadoop.job.ugi")); this.containerId = new XLearningContainerId(ConverterUtils.toContainerId(System .getenv(ApplicationConstants.Environment.CONTAINER_ID.name()))); this.downloadRetry = conf.getInt(XLearningConfiguration.XLEARNING_DOWNLOAD_FILE_RETRY, XLearningConfiguration.DEFAULT_XLEARNING_DOWNLOAD_FILE_RETRY);