} finally { fs.delete(DfsioeConfig.getInstance().getReportTmp(fsConfig), true); FileUtil.copyMerge(fs, DfsioeConfig.getInstance().getReportDir(fsConfig), fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), false, fsConfig, null); LOG.info("remote report file " + DfsioeConfig.getInstance().getReportTmp(fsConfig) + " merged."); BufferedReader lines = new BufferedReader(new InputStreamReader(new DataInputStream(fs.open(DfsioeConfig.getInstance().getReportTmp(fsConfig)))));
public boolean getMergeInHdfs(String src, String dest, JobConf conf) throws IllegalArgumentException, IOException { FileSystem fs = FileSystem.get(conf); Path srcPath = new Path(src); Path dstPath = new Path(dest); // Check if the path already exists if (!(fs.exists(srcPath))) { LOG.info("Path " + src + " does not exists!"); return false; } if (!(fs.exists(dstPath))) { LOG.info("Path " + dest + " does not exists!"); return false; } return FileUtil.copyMerge(fs, srcPath, fs, dstPath, false, conf, null); }
boolean deleteSource = false; String addString = null; result = FileUtil.copyMerge(fs, srcPath, fs, dstPath, deleteSource, conf, addString);
/** * merge all files in <code>sourceDir</code> into local <code>targetFile</code>, retrying on failure */ @Deprecated public static void copyMergeToLocal(String srcDir, String dstFile, int numTries, long delayBetweenTries) throws IOException { Configuration conf = new Configuration(); FileSystem hdfs = getFS(); FileSystem localfs = FileSystem.getLocal(conf); while (numTries-- > 0) { if (FileUtil.copyMerge(hdfs, new Path(srcDir), localfs, new Path(dstFile), false, conf, null)) { return; } try { Thread.sleep(delayBetweenTries); } catch (InterruptedException ie) { throw new RuntimeException(ie); } } throw new IOException("Could not copyMerge from \"" + srcDir + "\" to \"" + dstFile + "\"!"); }
/** * @param inputFiles a glob expression of the files to be merged * @param outputFile a destination file path * @param deleteSource delete source files after merging * @throws IOException */ private static Path mergeTextFiles(Configuration configuration, String inputFiles, String outputFile, boolean deleteSource, boolean deleteDestinationFileIfExist) throws IOException { JobConf conf = new JobConf(configuration, FileMerger.class); FileSystem fs = FileSystem.get(conf); Path inputPath = new Path(inputFiles); Path outputPath = new Path(outputFile); if (deleteDestinationFileIfExist) { if (fs.exists(outputPath)) { // carefully remove the destination file, not recursive fs.delete(outputPath, false); sLogger.info("Warning: remove destination file since it already exists..."); } } else { Preconditions.checkArgument(!fs.exists(outputPath), new IOException( "Destination file already exists...")); } FileUtil.copyMerge(fs, inputPath, fs, outputPath, deleteSource, conf, FILE_CONTENT_DELIMITER); sLogger.info("Successfully merge " + inputPath.toString() + " to " + outputFile); return outputPath; }
public void getmerge(String src, String localdst, boolean addnl) { Path srcPath = new Path(src); Path dst = new Path(localdst); try { FileSystem srcFs = getFS(srcPath); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); for (int i = 0; i < srcs.length; i++) { if (addnl) { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(configuration), dst, false, configuration, "\n"); } else { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(configuration), dst, false, configuration, null); } } } catch (IOException ex) { throw new HadoopException("Cannot getmerge " + ex.getMessage(), ex); } }
private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) { try { String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/')); File outputDest = File.createTempFile(fname, ".hdfs"); if (outputDest.exists()) { outputDest.delete(); } HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath); FileSystem fileSystem = hdfsInfo.getFileSystem(); FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf()); try { FileUtil.copyMerge( fileSystem, // src new Path(hdfsPath), FileSystem.getLocal(new Configuration()), // dest new Path(outputDest.toURI()), false, fileSystem.getConf(), null); } catch (IOException e) { return outputDest; } return new File(outputDest, fname); } catch (IOException ex) { throw new RuntimeCamelException(ex); } } },
private void distributeFiles() { try { URI[] uris = DistributedCache.getCacheFiles(conf); if (uris != null) { URI[] outURIs = new URI[uris.length]; for (int i = 0; i < uris.length; i++) { Path path = new Path(uris[i]); FileSystem fs = path.getFileSystem(conf); if (fs.isFile(path)) { outURIs[i] = uris[i]; } else { Path mergePath = new Path(path.getParent(), "sparkreadable-" + path.getName()); FileUtil.copyMerge(fs, path, fs, mergePath, false, conf, ""); outURIs[i] = mergePath.toUri(); } sparkContext.addFile(outURIs[i].toString()); } DistributedCache.setCacheFiles(outURIs, conf); } } catch (IOException e) { throw new RuntimeException("Error retrieving cache files", e); } }
private void distributeFiles() { try { URI[] uris = DistributedCache.getCacheFiles(conf); if (uris != null) { URI[] outURIs = new URI[uris.length]; for (int i = 0; i < uris.length; i++) { Path path = new Path(uris[i]); FileSystem fs = path.getFileSystem(conf); if (fs.isFile(path)) { outURIs[i] = uris[i]; } else { Path mergePath = new Path(path.getParent(), "sparkreadable-" + path.getName()); FileUtil.copyMerge(fs, path, fs, mergePath, false, conf, ""); outURIs[i] = mergePath.toUri(); } sparkContext.addFile(outURIs[i].toString()); } DistributedCache.setCacheFiles(outURIs, conf); } } catch (IOException e) { throw new RuntimeException("Error retrieving cache files", e); } }
/** * Get all the files in the directories that match the source file pattern * and merge and sort them to only one file on local fs * srcf is kept. * * Also adds a string between the files (useful for adding \n * to a text file) * @param srcf: a file pattern specifying source files * @param dstf: a destination local file/directory * @param endline: if an end of line character is added to a text file * @exception: IOException * @see org.apache.hadoop.fs.FileSystem.globStatus */ void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException { Path srcPath = new Path(srcf); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path [] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); for(int i=0; i<srcs.length; i++) { if (endline) { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(getConf()), dst, false, getConf(), "\n"); } else { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(getConf()), dst, false, getConf(), null); } } }
/** * Get all the files in the directories that match the source file pattern * and merge and sort them to only one file on local fs * srcf is kept. * * Also adds a string between the files (useful for adding \n * to a text file) * @param srcf: a file pattern specifying source files * @param dstf: a destination local file/directory * @param endline: if an end of line character is added to a text file * @exception: IOException * @see org.apache.hadoop.fs.FileSystem.globStatus */ void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException { Path srcPath = new Path(srcf); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path [] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); for(int i=0; i<srcs.length; i++) { if (endline) { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(getConf()), dst, false, getConf(), "\n"); } else { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(getConf()), dst, false, getConf(), null); } } }
/** * Calls FileUtil.copyMerge using the specified source and destination paths. * Both source and destination are assumed to be on the local file system. * The call will not delete source on completion and will not add an * additional string between files. * @param src String non-null source path. * @param dst String non-null destination path. * @return boolean true if the call to FileUtil.copyMerge was successful. * @throws IOException if an I/O error occurs. */ private boolean copyMerge(String src, String dst) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); final boolean result; try { Path srcPath = new Path(TEST_ROOT_DIR, src); Path dstPath = new Path(TEST_ROOT_DIR, dst); boolean deleteSource = false; String addString = null; result = FileUtil.copyMerge(fs, srcPath, fs, dstPath, deleteSource, conf, addString); } finally { fs.close(); } return result; }
/** * Calls FileUtil.copyMerge using the specified source and destination paths. * Both source and destination are assumed to be on the local file system. * The call will not delete source on completion and will not add an * additional string between files. * @param src String non-null source path. * @param dst String non-null destination path. * @return boolean true if the call to FileUtil.copyMerge was successful. * @throws IOException if an I/O error occurs. */ private boolean copyMerge(String src, String dst) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); final boolean result; try { Path srcPath = new Path(TEST_ROOT_DIR, src); Path dstPath = new Path(TEST_ROOT_DIR, dst); boolean deleteSource = false; String addString = null; result = FileUtil.copyMerge(fs, srcPath, fs, dstPath, deleteSource, conf, addString); } finally { fs.close(); } return result; }
FileUtil.copyMerge(pfs, partsPath, fs, outputPath, true, conf, null);
FileUtil.copyMerge(pfs, partsPath, fs, outputPath, true, conf, null);
FileUtil.copyMerge(pfs, partsPath, fs, outputPath, true, conf, null);