public FileSystem getRaw() { return getRawFileSystem(); }
/** * The src file is under this filesystem, and the dst is on the local disk. * Copy it from the remote filesystem to the local dst name. * delSrc indicates if the src will be removed * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem * as the local file system or not. RawLocalFileSystem is non checksumming, * So, It will not create any crc files at local. * * @param delSrc * whether to delete the src * @param src * path * @param dst * path * @param useRawLocalFileSystem * whether to use RawLocalFileSystem as local file system or not. * * @throws IOException for any IO error */ public void copyToLocalFile(boolean delSrc, Path src, Path dst, boolean useRawLocalFileSystem) throws IOException { Configuration conf = getConf(); FileSystem local = null; if (useRawLocalFileSystem) { local = getLocal(conf).getRawFileSystem(); } else { local = getLocal(conf); } FileUtil.copy(this, src, local, dst, delSrc, conf); }
if (!fs.isDirectory(src)) { // source is a file fs.copyToLocalFile(src, dst); FileSystem localFs = getLocal(getConf()).getRawFileSystem(); if (localFs.isDirectory(dst)) { dst = new Path(dst, src.getName());
final FileSystem lfs = FileSystem.getLocal(conf).getRawFileSystem();
public FileSystem getRaw() { return getRawFileSystem(); }
public FileSystem getRaw() { return getRawFileSystem(); }
public FileSystem getRaw() { return getRawFileSystem(); }
public FileSystem getRaw() { return getRawFileSystem(); }
private FileSystem getFileSystem() throws IOException { if (fs == null) { synchronized (this) { if (fs == null) { fs = FileSystem.get(hConfig); // TODO horrible! what worth is the FileSystem abstraction then? // not sure why this is, but the local file system's hflush() does // not appear to work. Using the raw local file system fixes it. if (fs instanceof LocalFileSystem) { fs = ((LocalFileSystem) fs).getRawFileSystem(); } } } } return fs; }
/** * test Syncable interface on raw local file system * @throws IOException */ public void testSyncable() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf).getRawFileSystem(); Path file = new Path(TEST_ROOT_DIR, "syncable"); FSDataOutputStream out = fs.create(file);; final int bytesWritten = 1; byte[] expectedBuf = new byte[] {'0', '1', '2', '3'}; try { out.write(expectedBuf, 0, 1); out.hflush(); verifyFile(fs, file, bytesWritten, expectedBuf); out.write(expectedBuf, bytesWritten, expectedBuf.length-bytesWritten); out.hsync(); verifyFile(fs, file, expectedBuf.length, expectedBuf); } finally { out.close(); } }
@Override protected FileSystem getLocalFS() throws IOException { return FileSystem.getLocal(getConf()).getRawFileSystem(); }
@Override protected FileSystem getLocalFS() throws IOException { return FileSystem.getLocal(getConf()).getRawFileSystem(); }
@Test public void testCorruptedCopyCrc() throws Exception { FSDataOutputStream out = lfs.getRawFileSystem().create(srcPath); out.writeChars("bang"); out.close(); shellRun(1, "-get", srcPath.toString(), dstPath.toString()); }
@Test public void testCorruptedCopyCrc() throws Exception { FSDataOutputStream out = lfs.getRawFileSystem().create(srcPath); out.writeChars("bang"); out.close(); shellRun(1, "-get", srcPath.toString(), dstPath.toString()); }
/** * test Syncable interface on raw local file system * @throws IOException */ @Test(timeout = 1000) public void testSyncable() throws IOException { FileSystem fs = fileSys.getRawFileSystem(); Path file = new Path(TEST_ROOT_DIR, "syncable"); FSDataOutputStream out = fs.create(file);; final int bytesWritten = 1; byte[] expectedBuf = new byte[] {'0', '1', '2', '3'}; try { out.write(expectedBuf, 0, 1); out.hflush(); verifyFile(fs, file, bytesWritten, expectedBuf); out.write(expectedBuf, bytesWritten, expectedBuf.length-bytesWritten); out.hsync(); verifyFile(fs, file, expectedBuf.length, expectedBuf); } finally { out.close(); } }
/** * test Syncable interface on raw local file system * @throws IOException */ @Test(timeout = 1000) public void testSyncable() throws IOException { FileSystem fs = fileSys.getRawFileSystem(); Path file = new Path(TEST_ROOT_DIR, "syncable"); FSDataOutputStream out = fs.create(file);; final int bytesWritten = 1; byte[] expectedBuf = new byte[] {'0', '1', '2', '3'}; try { out.write(expectedBuf, 0, 1); out.hflush(); verifyFile(fs, file, bytesWritten, expectedBuf); out.write(expectedBuf, bytesWritten, expectedBuf.length-bytesWritten); out.hsync(); verifyFile(fs, file, expectedBuf.length, expectedBuf); } finally { out.close(); } }
@Test public void testSetPermissionCrc() throws Exception { FileSystem rawFs = localFs.getRawFileSystem(); Path p = new Path(TEST_ROOT_DIR, "testCrcPermissions"); localFs.createNewFile(p); Path crc = localFs.getChecksumFile(p); assert(rawFs.exists(crc)); for (short mode : Arrays.asList((short)0666, (short)0660, (short)0600)) { FsPermission perm = new FsPermission(mode); localFs.setPermission(p, perm); assertEquals(perm, localFs.getFileStatus(p).getPermission()); assertEquals(perm, rawFs.getFileStatus(crc).getPermission()); } } }
@Test public void testSetPermissionCrc() throws Exception { FileSystem rawFs = localFs.getRawFileSystem(); Path p = new Path(TEST_ROOT_DIR, "testCrcPermissions"); localFs.createNewFile(p); Path crc = localFs.getChecksumFile(p); assert(rawFs.exists(crc)); for (short mode : Arrays.asList((short)0666, (short)0660, (short)0600)) { FsPermission perm = new FsPermission(mode); localFs.setPermission(p, perm); assertEquals(perm, localFs.getFileStatus(p).getPermission()); assertEquals(perm, rawFs.getFileStatus(crc).getPermission()); } } }
void verifyRename(Path srcPath, Path dstPath, boolean dstIsDir) throws Exception { localFs.delete(srcPath,true); localFs.delete(dstPath,true); Path realDstPath = dstPath; if (dstIsDir) { localFs.mkdirs(dstPath); realDstPath = new Path(dstPath, srcPath.getName()); } // ensure file + checksum are moved writeFile(localFs, srcPath, 1); assertTrue(localFs.exists(localFs.getChecksumFile(srcPath))); assertTrue(localFs.rename(srcPath, dstPath)); assertTrue(localFs.exists(localFs.getChecksumFile(realDstPath))); // create a file with no checksum, rename, ensure dst checksum is removed writeFile(localFs.getRawFileSystem(), srcPath, 1); assertFalse(localFs.exists(localFs.getChecksumFile(srcPath))); assertTrue(localFs.rename(srcPath, dstPath)); assertFalse(localFs.exists(localFs.getChecksumFile(realDstPath))); // create file with checksum, rename over prior dst with no checksum writeFile(localFs, srcPath, 1); assertTrue(localFs.exists(localFs.getChecksumFile(srcPath))); assertTrue(localFs.rename(srcPath, dstPath)); assertTrue(localFs.exists(localFs.getChecksumFile(realDstPath))); }
void verifyRename(Path srcPath, Path dstPath, boolean dstIsDir) throws Exception { localFs.delete(srcPath,true); localFs.delete(dstPath,true); Path realDstPath = dstPath; if (dstIsDir) { localFs.mkdirs(dstPath); realDstPath = new Path(dstPath, srcPath.getName()); } // ensure file + checksum are moved writeFile(localFs, srcPath, 1); assertTrue(localFs.exists(localFs.getChecksumFile(srcPath))); assertTrue(localFs.rename(srcPath, dstPath)); assertTrue(localFs.exists(localFs.getChecksumFile(realDstPath))); // create a file with no checksum, rename, ensure dst checksum is removed writeFile(localFs.getRawFileSystem(), srcPath, 1); assertFalse(localFs.exists(localFs.getChecksumFile(srcPath))); assertTrue(localFs.rename(srcPath, dstPath)); assertFalse(localFs.exists(localFs.getChecksumFile(realDstPath))); // create file with checksum, rename over prior dst with no checksum writeFile(localFs, srcPath, 1); assertTrue(localFs.exists(localFs.getChecksumFile(srcPath))); assertTrue(localFs.rename(srcPath, dstPath)); assertTrue(localFs.exists(localFs.getChecksumFile(realDstPath))); }