@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
String device = new DF(f, getConf()).getMount(); File parent = f.getParentFile(); File dir = null;
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
@Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { FileUtil.copy(this, src, this, dst, delSrc, getConf()); }
/** * Returns the local temporary directory. * @param localFileSystem the local file system * @return the output path (must be on local fs), or {@code null} if not defined * @throws IOException if failed to compute the path * @throws IllegalArgumentException if some parameters were {@code null} */ public static Path getLocalTemporaryDirectory(LocalFileSystem localFileSystem) throws IOException { if (localFileSystem == null) { throw new IllegalArgumentException("localFileSystem must not be null"); //$NON-NLS-1$ } Configuration conf = localFileSystem.getConf(); if (conf == null) { return null; } String path = conf.get(KEY_LOCAL_TEMPDIR); if (path == null) { return null; } LocalFileSystem fs = FileSystem.getLocal(conf); Path result = fs.makeQualified(new Path(path)); return result; }
public static OpenIntObjectHashMap<Vector> readMatrixByRowsFromDistributedCache(int numEntities, Configuration conf) throws IOException { IntWritable rowIndex = new IntWritable(); VectorWritable row = new VectorWritable(); OpenIntObjectHashMap<Vector> featureMatrix = numEntities > 0 ? new OpenIntObjectHashMap<Vector>(numEntities) : new OpenIntObjectHashMap<Vector>(); Path[] cachedFiles = HadoopUtil.getCachedFiles(conf); LocalFileSystem localFs = FileSystem.getLocal(conf); for (Path cachedFile : cachedFiles) { try (SequenceFile.Reader reader = new SequenceFile.Reader(localFs.getConf(), SequenceFile.Reader.file(cachedFile))) { while (reader.next(rowIndex, row)) { featureMatrix.put(rowIndex.get(), row.get()); } } } Preconditions.checkState(!featureMatrix.isEmpty(), "Feature matrix is empty"); return featureMatrix; }
/** * output without streaming. * @throws Exception if failed */ @Test public void output_nostreaming() throws Exception { profile.setOutputStreaming(false); profile.getLocalFileSystem().getConf().set( HadoopDataSourceUtil.KEY_LOCAL_TEMPDIR, localtemp.getPath()); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)) { output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(false)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
/** * output without streaming nor staging. * @throws Exception if failed */ @Test public void output_nomove() throws Exception { profile.setOutputStaging(false); profile.setOutputStreaming(false); profile.getLocalFileSystem().getConf().set( HadoopDataSourceUtil.KEY_LOCAL_TEMPDIR, localtemp.getPath()); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)) { output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(true)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
localFs.getChecksumFile(testPath),false,true,localFs.getConf()); assertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));