/** * Sync buffered data to DataNodes (flush to disk devices). * * @param syncFlags * Indicate the detailed semantic and actions of the hsync. * @throws IOException * @see FSDataOutputStream#hsync() */ public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException { OutputStream wrappedStream = getWrappedStream(); if (wrappedStream instanceof CryptoOutputStream) { wrappedStream.flush(); wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); } ((DFSOutputStream) wrappedStream).hsync(syncFlags); }
private void hsync(FSDataOutputStream fsOut) throws Exception { // 调用hsync时,必须设置SyncFlag.UPDATE_LENGTH,否则RDD或者MR任务读取不到写入的数据 // 参见: // https://issues.cloudera.org/browse/DISTRO-696; // http://www.hypertable.com/documentation/administrator_guide/hdfs_and_durability // https://blog.csdn.net/leen0304/article/details/77854052?locationNum=10&fps=1 // https://issues.apache.org/jira/browse/HDFS-11915 if (fsOut instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) fsOut).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else if (fsOut.getWrappedStream() instanceof DFSOutputStream) { ((DFSOutputStream) fsOut.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { fsOut.hsync(); } } }
/** * Sync buffered data to DataNodes (flush to disk devices). * * @param syncFlags * Indicate the detailed semantic and actions of the hsync. * @throws IOException * @see FSDataOutputStream#hsync() */ public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException { OutputStream wrappedStream = getWrappedStream(); if (wrappedStream instanceof CryptoOutputStream) { ((CryptoOutputStream) wrappedStream).flush(); wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); } ((DFSOutputStream) wrappedStream).hsync(syncFlags); }
/** * Sync buffered data to DataNodes (flush to disk devices). * * @param syncFlags * Indicate the detailed semantic and actions of the hsync. * @throws IOException * @see FSDataOutputStream#hsync() */ public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException { OutputStream wrappedStream = getWrappedStream(); if (wrappedStream instanceof CryptoOutputStream) { ((CryptoOutputStream) wrappedStream).flush(); wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); } ((DFSOutputStream) wrappedStream).hsync(syncFlags); }
/** * Similar with testRenameUCFileInSnapshot, but do renaming first and then * append file without closing it. Unit test for HDFS-5425. */ @Test public void testAppendFileAfterRenameInSnapshot() throws Exception { final Path test = new Path("/test"); final Path foo = new Path(test, "foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, test, "s0"); // rename bar --> bar2 final Path bar2 = new Path(foo, "bar2"); hdfs.rename(bar, bar2); // append file and keep it as underconstruction. FSDataOutputStream out = hdfs.append(bar2); out.writeByte(0); ((DFSOutputStream) out.getWrappedStream()).hsync( EnumSet.of(SyncFlag.UPDATE_LENGTH)); // save namespace and restart restartClusterAndCheckImage(true); }
/** * Test hsync (with updating block length in NameNode) while no data is * actually written yet */ @Test public void hSyncUpdateLength_00() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes( 2).build(); DistributedFileSystem fileSystem = cluster.getFileSystem(); try { Path path = new Path(fName); FSDataOutputStream stm = fileSystem.create(path, true, 4096, (short) 2, AppendTestUtil.BLOCK_SIZE); System.out.println("Created file " + path.toString()); ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet .of(SyncFlag.UPDATE_LENGTH)); long currentFileLength = fileSystem.getFileStatus(path).getLen(); assertEquals(0L, currentFileLength); stm.close(); } finally { fileSystem.close(); cluster.shutdown(); } }
((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet .of(SyncFlag.UPDATE_LENGTH)); DFSTestUtil.abortStream((DFSOutputStream) out.getWrappedStream()); ((DFSOutputStream) out2.getWrappedStream()).hsync(EnumSet .of(SyncFlag.UPDATE_LENGTH)); DFSTestUtil.abortStream((DFSOutputStream) out2.getWrappedStream());
AppendTestUtil.BLOCK_SIZE); System.out.println("Created file " + path.toString()); ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet .of(SyncFlag.END_BLOCK)); long currentFileLength = fileSystem.getFileStatus(path).getLen(); ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet .of(SyncFlag.END_BLOCK)); currentFileLength = fileSystem.getFileStatus(path).getLen(); ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet .of(SyncFlag.END_BLOCK)); currentFileLength = fileSystem.getFileStatus(path).getLen();
((DFSOutputStream)stm.getWrappedStream()).hsync(syncFlags); } else { ((DFSOutputStream)stm.getWrappedStream()).hflush();
String appendContent = "appending-content"; out.writeBytes(appendContent); ((DFSOutputStream) out.getWrappedStream()).hsync( EnumSet.of(SyncFlag.UPDATE_LENGTH));
((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet .of(SyncFlag.UPDATE_LENGTH));
((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
.getWrappedStream(); streams[i].write("1".getBytes()); streams[i].hsync();
.getWrappedStream(); streams[i].write("1".getBytes()); streams[i].hsync();