Refine search
/** * @param fs File system. * @param src Source path. * @param dst Destination path. * @return Path to file to hdfs file system. */ public static Path copyLocalToHdfs(FileSystem fs, String src, String dst) throws Exception { Path dstPath = new Path(dst); // Local file isn't removed, dst file override. fs.copyFromLocalFile(false, true, new Path(src), dstPath); return dstPath; }
@Override public void copyFromLocalFile(File srcFile, URI dstUri) throws Exception { _hadoopFS.copyFromLocalFile(new Path(srcFile.toURI()), new Path(dstUri)); }
private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap, Path destDir) throws IOException { for (String localFilePath : SPLITTER.split(localFilePathList)) { Path srcFilePath = new Path(localFilePath); Path destFilePath = new Path(destDir, srcFilePath.getName()); this.fs.copyFromLocalFile(srcFilePath, destFilePath); if (resourceMap.isPresent()) { YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get()); } } }
private void addAppJars(String jarFilePathList, Optional<Map<String, LocalResource>> resourceMap, Path destDir) throws IOException { for (String jarFilePath : SPLITTER.split(jarFilePathList)) { Path srcFilePath = new Path(jarFilePath); Path destFilePath = new Path(destDir, srcFilePath.getName()); this.fs.copyFromLocalFile(srcFilePath, destFilePath); if (resourceMap.isPresent()) { YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get()); } } }
/** * Copy file in DFS from p to newPath * @param fs file system * @param p old path * @param newPath new path * @throws IOException exception */ protected void copyFile(FileSystem fs, Path p, Path newPath) throws IOException { File f = File.createTempFile("data", "meta"); Path localPath = new Path(f.getAbsolutePath()); fs.copyToLocalFile(p, localPath); fs.copyFromLocalFile(localPath, newPath); boolean exists = fs.exists(newPath); if (!exists) { throw new IOException("Failed to copy meta file to: "+ newPath); } }
private void addLibJars(Path srcLibJarDir, Optional<Map<String, LocalResource>> resourceMap, Path destDir) throws IOException { FileSystem localFs = FileSystem.getLocal(this.yarnConfiguration); FileStatus[] libJarFiles = localFs.listStatus(srcLibJarDir); if (libJarFiles == null || libJarFiles.length == 0) { return; } for (FileStatus libJarFile : libJarFiles) { Path destFilePath = new Path(destDir, libJarFile.getPath().getName()); this.fs.copyFromLocalFile(libJarFile.getPath(), destFilePath); if (resourceMap.isPresent()) { YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get()); } } }
private void copyConfig(FileSystem lfs, Path confPath, String f) throws IOException { HiveConf.getBoolVar(new Configuration(false), ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS); // they will be file:// URLs lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); }
public void copyToDFSDirecory(FileSystem destFs, Path destPath) throws IOException, HiveException { if (addCursor > 0) { this.spillBlock(this.currentWriteBlock, addCursor); } if (tempOutPath == null || tempOutPath.toString().trim().equals("")) { return; } this.closeWriter(); LOG.info("RowContainer copied temp file " + tmpFile.getAbsolutePath() + " to dfs directory " + destPath.toString()); destFs .copyFromLocalFile(true, tempOutPath, new Path(destPath, new Path(tempOutPath.getName()))); clearRows(); }
public void copyToDFSDirecory(FileSystem destFs, Path destPath) throws IOException, HiveException { if (addCursor > 0) { this.spillBlock(this.currentWriteBlock, addCursor); } if (tempOutPath == null || tempOutPath.toString().trim().equals("")) { return; } this.closeWriter(); LOG.info("RowContainer copied temp file " + tmpFile.getAbsolutePath() + " to dfs directory " + destPath.toString()); destFs .copyFromLocalFile(true, tempOutPath, new Path(destPath, new Path(tempOutPath.getName()))); clearRows(); }
private void localizeJarForClass(FileSystem lfs, Path libDir, String className, boolean doThrow) throws IOException { String jarPath = null; boolean hasException = false; try { Class<?> auxClass = Class.forName(className); jarPath = Utilities.jarFinderGetJar(auxClass); } catch (Throwable t) { if (doThrow) { throw (t instanceof IOException) ? (IOException)t : new IOException(t); } hasException = true; String err = "Cannot find a jar for [" + className + "] due to an exception (" + t.getMessage() + "); not packaging the jar"; LOG.error(err); System.err.println(err); } if (jarPath != null) { lfs.copyFromLocalFile(new Path(jarPath), libDir); } else if (!hasException) { String err = "Cannot find a jar for [" + className + "]; not packaging the jar"; if (doThrow) { throw new IOException(err); } LOG.error(err); System.err.println(err); } }
protected Path uploadFileIfNecessary(FileSystem localFS, Path localPath, FileSystem destFS) throws IOException { // Fast path for local FS -- DistributedCache + local JobRunner seems copy/link files automatically if (destFS.equals(localFS)) { log.debug("Skipping file upload for {} (destination filesystem {} equals local filesystem)", localPath, destFS); return localPath; } Path destPath = new Path(destFS.getHomeDirectory() + "/" + HDFS_TMP_LIB_DIR + "/" + localPath.getName()); Stats fileStats = null; try { fileStats = compareModtimes(localFS, localPath, destFS, destPath); } catch (IOException e) { log.warn("Unable to read or stat file: localPath={}, destPath={}, destFS={}", localPath, destPath, destFS); } if (!fileStats.isRemoteCopyCurrent()) { log.debug("Copying {} to {}", localPath, destPath); destFS.copyFromLocalFile(localPath, destPath); if (null != fileStats.local) { final long mtime = fileStats.local.getModificationTime(); log.debug("Setting modtime on {} to {}", destPath, mtime); destFS.setTimes(destPath, mtime, -1); // -1 means leave atime alone } } return destPath; }
/** * Add local non-jar files the job depends on to DistributedCache. */ @SuppressWarnings("deprecation") private void addLocalFiles(Path jobFileDir, String jobFileList, Configuration conf) throws IOException { DistributedCache.createSymlink(conf); for (String jobFile : SPLITTER.split(jobFileList)) { Path srcJobFile = new Path(jobFile); // DistributedCache requires absolute path, so we need to use makeQualified. Path destJobFile = new Path(this.fs.makeQualified(jobFileDir), srcJobFile.getName()); // Copy the file from local file system to HDFS this.fs.copyFromLocalFile(srcJobFile, destJobFile); // Create a URI that is in the form path#symlink URI destFileUri = URI.create(destJobFile.toUri().getPath() + "#" + destJobFile.getName()); LOG.info(String.format("Adding %s to DistributedCache", destFileUri)); // Finally add the file to DistributedCache with a symlink named after the file name DistributedCache.addCacheFile(destFileUri, conf); } }
Path jarPath = new Path(Utilities.jarFinderGetJar(c)); lfs.copyFromLocalFile(jarPath, libDir); if (LOG.isDebugEnabled()) { LOG.debug("Copying " + jarPath + " to " + libDir);
+ "/" + localSrcPath.getName(); Path dst = new Path(homedir, suffix); fs.copyFromLocalFile(false, true, localSrcPath, dst);
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
/** * Copy dependency jars from local fs to HDFS. */ private void copyDependencyJarsToHdfs() throws IOException { if (!this.state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) { return; } LocalFileSystem lfs = FileSystem.getLocal(this.conf); Path tmpJarFileDir = new Path(this.tmpOutputDir, "_gobblin_compaction_jars"); this.state.setProp(COMPACTION_JARS, tmpJarFileDir.toString()); this.fs.delete(tmpJarFileDir, true); for (String jarFile : this.state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) { for (FileStatus status : lfs.globStatus(new Path(jarFile))) { Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName()); this.fs.copyFromLocalFile(status.getPath(), tmpJarFile); LOG.info(String.format("%s will be added to classpath", tmpJarFile)); } } }
/** * Uploads a local file to HDFS * This method is not thread safe * * @param source * @param conf * @return * @throws IOException */ public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException { Path localFile = new Path(source.getPath()); Path remoteFile = new Path(SessionState.get().getSparkSession().getHDFSSessionDir(), getFileName(source)); FileSystem fileSystem = FileSystem.get(remoteFile.toUri(), conf); // Overwrite if the remote file already exists. Whether the file can be added // on executor is up to spark, i.e. spark.files.overwrite fileSystem.copyFromLocalFile(false, true, localFile, remoteFile); Path fullPath = fileSystem.getFileStatus(remoteFile).getPath(); return fullPath.toUri(); }
/** * @param version Ignite version. * @return Ignite. */ public Path getIgnite(String version) throws Exception { checkDownloadFolder(); // Download ignite. String fileName = downloadIgnite(version); Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName); if (!fs.exists(dst)) fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst); return dst; }
@Override public String put(final StorageMetadata metadata, final File localFile) { this.hdfsAuth.authorize(); final Path projectsPath = new Path(this.rootUri.getPath(), String.valueOf(metadata.getProjectId())); try { if (this.hdfs.mkdirs(projectsPath)) { log.info("Created project dir: " + projectsPath); } final Path targetPath = createTargetPath(metadata, projectsPath); if (this.hdfs.exists(targetPath)) { log.info( String.format("Duplicate Found: meta: %s path: %s", metadata, targetPath)); return getRelativePath(targetPath); } // Copy file to HDFS log.info(String.format("Creating project artifact: meta: %s path: %s", metadata, targetPath)); this.hdfs.copyFromLocalFile(new Path(localFile.getAbsolutePath()), targetPath); return getRelativePath(targetPath); } catch (final IOException e) { log.error("error in put(): Metadata: " + metadata); throw new StorageException(e); } }
/** * Copy dependent jars to a temporary job directory on HDFS */ private void copyJarDependencies (State state) throws IOException { if (this.tmpJobDir == null) { throw new RuntimeException("Job directory is not created"); } if (!state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) { return; } // create sub-dir to save jar files LocalFileSystem lfs = FileSystem.getLocal(HadoopUtils.getConfFromState(state)); Path tmpJarFileDir = new Path(this.tmpJobDir, MRCompactor.COMPACTION_JAR_SUBDIR); this.fs.mkdirs(tmpJarFileDir); state.setProp (MRCompactor.COMPACTION_JARS, tmpJarFileDir.toString()); // copy jar files to hdfs for (String jarFile : state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) { for (FileStatus status : lfs.globStatus(new Path(jarFile))) { Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName()); this.fs.copyFromLocalFile(status.getPath(), tmpJarFile); log.info(String.format("%s will be added to classpath", tmpJarFile)); } } } }