out.println(resource.getType().toString()); out.print(" Visiblity: "); out.println(resource.getVisibility().toString());
@Override public LocalResourceVisibility getVisibility() { return localResource.getVisibility(); }
@Override public LocalResourceVisibility getVisibility() { return localResource.getVisibility(); }
@Override public LocalResourceVisibility getVisibility() { return localResource.getVisibility(); }
private void checkLocalResources(Map<String, LocalResource> localResources) { for (Map.Entry<String, LocalResource> rsrcEntry : localResources .entrySet()) { if (rsrcEntry.getValue() == null || rsrcEntry.getValue().getResource() == null) { throw new NullPointerException( "Null resource URL for local resource " + rsrcEntry.getKey() + " : " + rsrcEntry.getValue()); } else if (rsrcEntry.getValue().getType() == null) { throw new NullPointerException( "Null resource type for local resource " + rsrcEntry.getKey() + " : " + rsrcEntry.getValue()); } else if (rsrcEntry.getValue().getVisibility() == null) { throw new NullPointerException( "Null resource visibility for local resource " + rsrcEntry.getKey() + " : " + rsrcEntry.getValue()); } } }
Callable<Path> download(Path destDirPath, LocalResource rsrc, UserGroupInformation ugi) throws IOException { // For private localization FsDownload creates folder in destDirPath. Parent // directories till user filecache folder is created here. if (rsrc.getVisibility() == LocalResourceVisibility.PRIVATE) { createParentDirs(destDirPath); } diskValidator.checkStatus(new File(destDirPath.toUri().getRawPath())); return new FSDownloadWrapper(lfs, ugi, conf, destDirPath, rsrc); }
Callable<Path> download(Path destDirPath, LocalResource rsrc, UserGroupInformation ugi) throws IOException { // For private localization FsDownload creates folder in destDirPath. Parent // directories till user filecache folder is created here. if (rsrc.getVisibility() == LocalResourceVisibility.PRIVATE) { createParentDirs(destDirPath); } DiskChecker.checkDir(new File(destDirPath.toUri().getRawPath())); return new FSDownload(lfs, ugi, conf, destDirPath, rsrc); }
private Path copy(Path sCopy, Path dstdir) throws IOException { FileSystem sourceFs = sCopy.getFileSystem(conf); Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName()); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + " changed on src filesystem (expected " + resource.getTimestamp() + ", was " + sStat.getModificationTime()); } if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { if (!isPublic(sourceFs, sCopy, sStat, statCache)) { throw new IOException("Resource " + sCopy + " is not publicly accessable and as such cannot be part of the" + " public cache."); } } FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false, true, conf); return dCopy; }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private Path getPathForLocalization(LocalResource rsrc, LocalResourcesTracker tracker) throws IOException, URISyntaxException { String user = context.getUser(); ApplicationId appId = context.getContainerId().getApplicationAttemptId().getApplicationId(); LocalResourceVisibility vis = rsrc.getVisibility(); String cacheDirectory = null; if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only cacheDirectory = getUserFileCachePath(user); } else {// APPLICATION ONLY cacheDirectory = getAppFileCachePath(user, appId.toString()); } Path dirPath = dirsHandler.getLocalPathForWrite(cacheDirectory, ContainerLocalizer.getEstimatedSize(rsrc), false); return tracker.getPathForLocalization(new LocalResourceRequest(rsrc), dirPath, delService); }
private Path getPathForLocalization(LocalResource rsrc, LocalResourcesTracker tracker) throws IOException, URISyntaxException { String user = context.getUser(); ApplicationId appId = context.getContainerId().getApplicationAttemptId().getApplicationId(); LocalResourceVisibility vis = rsrc.getVisibility(); String cacheDirectory = null; if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only cacheDirectory = getUserFileCachePath(user); } else {// APPLICATION ONLY cacheDirectory = getAppFileCachePath(user, appId.toString()); } Path dirPath = dirsHandler.getLocalPathForWrite(cacheDirectory, ContainerLocalizer.getEstimatedSize(rsrc), false); return tracker.getPathForLocalization(new LocalResourceRequest(rsrc), dirPath, delService); }
public void testCreateLocalResourceForConfigurationFile() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); String filename = "foo.xml"; Configuration conf = has.createConfiguration(getNameNodeUri()); conf.set("foo", "bar"); LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(), getFsTestCaseDir()); assertNotNull(lRes); assertEquals(LocalResourceType.FILE, lRes.getType()); assertEquals(LocalResourceVisibility.APPLICATION, lRes.getVisibility()); Path resPath = ConverterUtils.getPathFromYarnURL(lRes.getResource()); assertEquals(new Path(getFsTestCaseDir(), "foo.xml"), resPath); Configuration conf2 = new Configuration(false); conf2.addResource(getFileSystem().open(resPath)); assertEquals("bar", conf2.get("foo")); } }
private Path getPathForLocalization(LocalResource rsrc) throws IOException, URISyntaxException { String user = context.getUser(); ApplicationId appId = context.getContainerId().getApplicationAttemptId().getApplicationId(); LocalResourceVisibility vis = rsrc.getVisibility(); LocalResourcesTracker tracker = getLocalResourcesTracker(vis, user, appId); String cacheDirectory = null; if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only cacheDirectory = getUserFileCachePath(user); } else {// APPLICATION ONLY cacheDirectory = getAppFileCachePath(user, appId.toString()); } Path dirPath = dirsHandler.getLocalPathForWrite(cacheDirectory, ContainerLocalizer.getEstimatedSize(rsrc), false); return tracker.getPathForLocalization(new LocalResourceRequest(rsrc), dirPath, delService); }