/** * Append to an existing file (optional operation). * Same as append(f, bufferSize, null). * @param f the existing file to be appended. * @param bufferSize the size of the buffer to be used. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public FSDataOutputStream append(Path f, int bufferSize) throws IOException { return append(f, bufferSize, null); }
@Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { return fs.append(f, bufferSize, progress); }
private FileLock(FileSystem fs, Path lockFile, String spoutId, LogEntry entry) throws IOException { this.fs = fs; this.lockFile = lockFile; this.lockFileStream = fs.append(lockFile); this.componentID = spoutId; LOG.info("Acquired abandoned lockFile {}, Spout {}", lockFile, spoutId); logProgress(entry.fileOffset, true); }
/** * Append to an existing file (optional operation). * Same as * {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, * IO_FILE_BUFFER_SIZE_DEFAULT), null)} * @param f the existing file to be appended. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public FSDataOutputStream append(Path f) throws IOException { return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), null); }
/** * Open a BufferedWriter * @param errFilePath path to write the file */ public void open(Path errFilePath) throws IOException { this.fs.mkdirs(errFilePath.getParent()); OutputStream os = this.closer.register(this.fs.exists(errFilePath) ? this.fs.append(errFilePath) : this.fs.create(errFilePath)); this.writer = this.closer .register(new BufferedWriter(new OutputStreamWriter(os, ConfigurationKeys.DEFAULT_CHARSET_ENCODING))); }
private OutputStream openWriterAppendMode() { try { // not sure if the exception was due to a incorrect state within the writer hence closing it close(); return FileSystem.get(hiveConf).append(writePath); } catch (IOException e1) { String message = "there was an error to open the file {} in append mode"; LOG.error(message, writePath.toString(), e1); throw new IllegalStateException(message, e1); } }
@Override public Object call() throws Exception { return fs.append(path); } }, IOException.class, "File system is stopped.");
@Override public Object call() throws Exception { return fs.append(file); } }, IOException.class, null);
@Override public Object call() throws Exception { return fs.append(null); } }, NullPointerException.class, "Ouch! Argument cannot be null: f");
public FSDataOutputStream append(Path f, int bufferSize) throws java.io.IOException { return this.underlyingFs.append(replaceScheme(f, this.replacementScheme, this.underlyingScheme), bufferSize); }
public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws java.io.IOException { return this.underlyingFs.append(replaceScheme(f, this.replacementScheme, this.underlyingScheme), bufferSize, progress); }
@Override public Object call() throws Exception { return fs.append(new Path(fsHome, dir), 1024); } }, IOException.class, null);
@Override public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException { LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0); return ctx.getFs().append(ctx.path, arg1, arg2); }
@Override public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress) throws IOException { InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(f), true); return res.targetFileSystem.append(res.remainingPath, bufferSize, progress); }
/** {@inheritDoc} */ @Override public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props) { try { Path hadoopPath = convert(path); FileSystem fs = fileSystemForUser(); if (create && !fs.exists(hadoopPath)) return fs.create(hadoopPath, false, bufSize); else return fs.append(convert(path), bufSize); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]"); } }
/** {@inheritDoc} */ @Override public OutputStream openOutputStream(String path, boolean append) throws IOException { Path p = new Path(path); if (append) return get().append(p); else return get().create(p, true/*overwrite*/); }
/** @throws Exception If failed. */ @Test public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception { Path fsHome = new Path(primaryFsUri); final Path file = new Path(fsHome, "someFile"); FSDataOutputStream os = fs.create(file); os.close(); FSDataOutputStream appendOs = fs.append(file); GridTestUtils.assertThrows(log, new Callable<Object>() { @Override public Object call() throws Exception { return fs.append(file); } }, IOException.class, null); appendOs.close(); }
@Test public void testMovedWALDuringRecovery() throws Exception { // This partial mock will throw LEE for every file simulating // files that were moved FileSystem spiedFs = Mockito.spy(fs); // The "File does not exist" part is very important, // that's how it comes out of HDFS Mockito.doThrow(new LeaseExpiredException("Injected: File does not exist")). when(spiedFs).append(Mockito.<Path>any()); retryOverHdfsProblem(spiedFs); }
@Test public void testAppendSemantics() throws Exception { //1 try to append to an open file Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1"); try (FSDataOutputStream os1 = fs.create(file1, false)) { fs.append(file1); // should fail fail("Append did not throw an exception"); } catch (RemoteException e) { // expecting AlreadyBeingCreatedException inside RemoteException Assert.assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass()); } //2 try to append to a closed file try (FSDataOutputStream os2 = fs.append(file1)) { assertThat(os2, notNull()); } }
HadoopRecoverableFsDataOutputStream( FileSystem fs, HadoopFsRecoverable recoverable) throws IOException { ensureTruncateInitialized(); this.fs = checkNotNull(fs); this.targetFile = checkNotNull(recoverable.targetFile()); this.tempFile = checkNotNull(recoverable.tempFile()); // truncate back and append try { truncate(fs, tempFile, recoverable.offset()); } catch (Exception e) { throw new IOException("Missing data in tmp file: " + tempFile, e); } waitUntilLeaseIsRevoked(tempFile); out = fs.append(tempFile); // sanity check long pos = out.getPos(); if (pos != recoverable.offset()) { IOUtils.closeQuietly(out); throw new IOException("Truncate failed: " + tempFile + " (requested=" + recoverable.offset() + " ,size=" + pos + ')'); } }