/** * Dump a string to a file * * @param location where to write to * @param data the data to write * @throws IOException on any error */ public void dump(File location, String data) throws IOException { File parent = location.getParentFile(); if (!parent.exists()) { forceMkdir(parent); } try (Writer w = getWriter(location)) { w.write(data); } }
protected void extractDirFromJar(String jarpath, String dir, Path dest) throws IOException { LOG.debug("EXTRACTING {} from {} and placing it at {}", dir, jarpath, dest); if (!Files.exists(dest)) { //Create the directory no matter what. This is so we can check if it was downloaded in the future. Files.createDirectories(dest); } try (JarFile jarFile = new JarFile(jarpath)) { String toRemove = dir + '/'; Enumeration<JarEntry> jarEnums = jarFile.entries(); while (jarEnums.hasMoreElements()) { JarEntry entry = jarEnums.nextElement(); String name = entry.getName(); if (!entry.isDirectory() && name.startsWith(toRemove)) { String shortenedName = name.substring(toRemove.length()); Path targetFile = dest.resolve(shortenedName); LOG.debug("EXTRACTING {} SHORTENED to {} into {}", name, shortenedName, targetFile); fsOps.forceMkdir(targetFile.getParent()); try (FileOutputStream out = new FileOutputStream(targetFile.toFile()); InputStream in = jarFile.getInputStream(entry)) { IOUtils.copy(in, out); } } else { LOG.debug("Skipping {}", entry); } } } }
/** * Setup the container to run. By default this creates the needed directories/links in the local file system PREREQUISITE: All needed * blobs and topology, jars/configs have been downloaded and placed in the appropriate locations * * @throws IOException on any error */ protected void setup() throws IOException { _type.assertFull(); if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) { LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment, _supervisorId, _port, _workerId); throw new IllegalStateException("Not all needed files are here!!!!"); } LOG.info("Setting up {}:{}", _supervisorId, _workerId); _ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId))); File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (!_ops.fileExists(workerArtifacts)) { _ops.forceMkdir(workerArtifacts); _ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts); } String user = getWorkerUser(); writeLogMetadata(user); saveWorkerUser(user); createArtifactsLink(); createBlobstoreLinks(); }
v -> { Path path = topologyBasicBlobsRootDir.resolve(type.getTempFileName(v)); fsOps.forceMkdir(path.getParent()); return path; }, fsOps::getOutputStream);
File userDir = getLocalUserFileCacheDir(topoOwner); if (!fsOps.fileExists(userDir)) { fsOps.forceMkdir(userDir);
verify(ops).forceMkdir(new File(workerRoot, "pids")); verify(ops).forceMkdir(new File(workerRoot, "tmp")); verify(ops).forceMkdir(new File(workerRoot, "heartbeats")); verify(ops).fileExists(workerArtifacts);
verify(ops).forceMkdir(userDir);
/** * Dump a string to a file * @param location where to write to * @param data the data to write * @throws IOException on any error */ public void dump(File location, String data) throws IOException { File parent = location.getParentFile(); if (!parent.exists()) { forceMkdir(parent); } try (Writer w = getWriter(location)) { w.write(data); } }
_ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId))); _ops.forceMkdir(workerArtifacts); _ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts);
@Override protected void downloadBaseBlobs(File tmproot) throws Exception { _fsOps.forceMkdir(tmproot); String stormCodeKey = ConfigUtils.masterStormCodeKey(_topologyId); String stormConfKey = ConfigUtils.masterStormConfKey(_topologyId);
File userDir = _localizer.getLocalUserFileCacheDir(topoOwner); if (!_fsOps.fileExists(userDir)) { _fsOps.forceMkdir(userDir);
protected void downloadBaseBlobs(File tmproot) throws Exception { String stormJarKey = ConfigUtils.masterStormJarKey(_topologyId); String stormCodeKey = ConfigUtils.masterStormCodeKey(_topologyId); String stormConfKey = ConfigUtils.masterStormConfKey(_topologyId); String jarPath = ConfigUtils.supervisorStormJarPath(tmproot.getAbsolutePath()); String codePath = ConfigUtils.supervisorStormCodePath(tmproot.getAbsolutePath()); String confPath = ConfigUtils.supervisorStormConfPath(tmproot.getAbsolutePath()); _fsOps.forceMkdir(tmproot); _fsOps.restrictDirectoryPermissions(tmproot); ClientBlobStore blobStore = Utils.getClientBlobStoreForSupervisor(_conf); try { Utils.downloadResourcesAsSupervisor(stormJarKey, jarPath, blobStore); Utils.downloadResourcesAsSupervisor(stormCodeKey, codePath, blobStore); Utils.downloadResourcesAsSupervisor(stormConfKey, confPath, blobStore); } finally { blobStore.shutdown(); } Utils.extractDirFromJar(jarPath, ConfigUtils.RESOURCES_SUBDIR, tmproot); }