@Override public FileStatus next() throws IOException { if (!hasNext) { throw new NoSuchElementException(); } hasNext = false; return getFileStatus(absolutePath); } };
@Override public void initialize(URI name, Configuration conf) throws IOException { initialize(name, conf, newLocalFileSystem(conf, config.isLocalAccessAllowed())); }
@Override public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException { final RemoteIterator<FileStatus> remoteIterator = listStatusIterator(f); // Note: RemoteIterator has no relation to java.util.Iterator so frequently used // helper functions can't be called on them. final List<FileStatus> statuses = Lists.newArrayList(); while (remoteIterator.hasNext()) { statuses.add(remoteIterator.next()); } return statuses.toArray(new FileStatus[0]); }
@Override public void setWorkingDirectory(Path newDir) { Path absolutePath = toAbsolutePath(newDir); checkPath(absolutePath); this.workingDirectory = absolutePath; }
@Override public Path canonicalizePath(Path p) throws IOException { Path absolutePath = toAbsolutePath(p); checkPath(absolutePath); if (isRemoteFile(absolutePath)) { return absolutePath; } if (isDirectory(absolutePath)) { return absolutePath; } if (localAccessAllowed) { return createRemotePath(localAddress, absolutePath); } // We aren't local or remote. That means we are a client node that wants to create a file. // We'll pick a random external location. final NodeEndpoint randomDelegate = getRandomDelegate(); return createRemotePath(randomDelegate.getAddress(), absolutePath); }
@Override public FileStatus getFileStatus(Path f) throws IOException { Path absolutePath = toAbsolutePath(f); checkPath(absolutePath); // if the path is not a remote file path if (!isRemoteFile(absolutePath)) { return new GetFileStatusTask(absolutePath).get(); } // Parse top level directory try { RemotePath remotePath = getRemotePath(absolutePath); FileSystem delegate = getDelegateFileSystem(remotePath.address); FileStatus status = delegate.getFileStatus(remotePath.path); return fixFileStatus(remotePath.address, status); } catch (IllegalArgumentException e) { throw (FileNotFoundException) (new FileNotFoundException("No file " + absolutePath).initCause(e)); } }
@Override public RemoteIterator<FileStatus> listStatusIterator(Path f) throws FileNotFoundException, IOException { final Path absolutePath = toAbsolutePath(f); checkPath(absolutePath); if (isRemoteFile(absolutePath)) { return new RemoteIterator<FileStatus>() { private boolean hasNext = true; @Override public boolean hasNext() throws IOException { return hasNext; } @Override public FileStatus next() throws IOException { if (!hasNext) { throw new NoSuchElementException(); } hasNext = false; return getFileStatus(absolutePath); } }; } return new ListStatusIteratorTask(absolutePath).get(); }
/** * Creates a pseudo distributed filesystem with the provided context * * @param context * @return * @throws IOException */ private PseudoDistributedFileSystem newPseudoDistributedFileSystem(PDFSConfig config) throws IOException { PseudoDistributedFileSystem fs = new PseudoDistributedFileSystem(config) { @Override FileSystem newRemoteFileSystem(NodeEndpoint endpoint) throws IOException { assertEquals("10.0.0.2", endpoint.getAddress()); return mockRemoteFS; } }; fs.setConf(hadoopConf); fs.initialize(URI.create("pdfs:///"), hadoopConf, mockLocalFS); return fs; }
@Test public void testDeleteFile() throws IOException { doReturn(true).when(mockRemoteFS).delete( new Path("/foo/bar"), false); Path path = new Path("/foo/10.0.0.2@bar"); assertTrue(fs.delete(path, false)); }
@Test public void testCanonicalizeLocalFile() throws IOException { Path path = new Path("/foo/bar/file"); Path resolvedPath = fs.canonicalizePath(path); assertEquals(new Path("/foo/bar/10.0.0.1@file"), resolvedPath); }
@Test public void testAppendLocalFile() throws IOException { try { Path path = new Path("/foo/bar/file"); @SuppressWarnings("unused") FSDataOutputStream fdos = fs.append(path, 4096, null); fail("Expected append call to throw an exception"); } catch (IOException e) { // ok } }
@Test public void testCreateFile() throws IOException { FSDataOutputStream mockFDOS = mock(FSDataOutputStream.class); doReturn(mockFDOS).when(mockLocalFS).create( new Path("/foo/bar/file"), FsPermission.getFileDefault(), true, 0, (short) 1, 4096L,null); Path path = new Path("/foo/bar/10.0.0.1@file"); FSDataOutputStream fdos = fs.create(path, FsPermission.getFileDefault(), true, 0, (short) 1, 4096, null); assertNotNull(fdos); }
@Override public void start() throws Exception { FabricService fabricService = this.fabricService.get(); pool = new CloseableThreadPool("pdfs"); FabricRunnerFactory factory = fabricService.registerProtocol(PDFSProtocol.newInstance(identityProvider.get(), this.config, allocator, allowLocalAccess)); final PDFSConfig config = new PDFSConfig(pool, factory, allocator, nodeProvider, identityProvider.get(), allowLocalAccess); PseudoDistributedFileSystem.configure(config); // A hack until DX-4639 is addressed. createFileSystem(); }
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Path absolutePath = toAbsolutePath(f); checkPath(absolutePath); // Handle root if (absolutePath.isRoot()) { throw new AccessControlException("Cannot open " + f); } try { RemotePath remotePath = getRemotePath(absolutePath); FileSystem delegate = getDelegateFileSystem(remotePath.address); return delegate.open(remotePath.path, bufferSize); } catch (IllegalArgumentException e) { throw (FileNotFoundException) (new FileNotFoundException("No file " + absolutePath).initCause(e)); } }
@Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { Path absolutePath = toAbsolutePath(f); checkPath(absolutePath); // Handle root if (absolutePath.isRoot()) { // Path always exists return true; } if (isRemoteFile(absolutePath)) { // Attempting to create a subdirectory for a file throw new IOException("Cannot create a directory under file " + f); } return new MkdirsTask(absolutePath, permission).get(); }
@Test public void testDeleteUnknownLocalFile() throws IOException { doThrow(FileNotFoundException.class).when(mockLocalFS).delete( new Path("/foo/unknown"), false); Path path = new Path("/foo/10.0.0.1@unknown"); try{ fs.delete(path, false); fail("Expecting FileNotFoundException"); } catch(FileNotFoundException e) { // nothing } }
@Test public void testCanonicalizeRemoteFile() throws IOException { Path path = new Path("/foo/bar/10.0.0.2@file"); Path resolvedPath = fs.canonicalizePath(path); assertEquals(new Path("/foo/bar/10.0.0.2@file"), resolvedPath); }
@Test public void testAppendFile() throws IOException { FSDataOutputStream mockFDOS = mock(FSDataOutputStream.class); doReturn(mockFDOS).when(mockRemoteFS).append( new Path("/foo/bar/file"), 4096, null); Path path = new Path("/foo/bar/10.0.0.2@file"); FSDataOutputStream fdos = fs.append(path, 4096, null); assertNotNull(fdos); }
@Test public void testCreateRoot() throws IOException { Path root = new Path("/"); try { fs.create(root, FsPermission.getFileDefault(), true, 0, (short) 1, 4096, null); fail("Expected create call to throw an exception"); } catch (AccessControlException e) { // ok } }
/** * Create a new file. Three possibilities: * - This is a data node and you're trying to create a unqualified file => write locally. * - This is a client node and you're trying to create unqualified file => pick a random data node and write there. * - The path you provide is qualified => write to that node. */ @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { final Path absolutePath = toAbsolutePath(f); checkPath(absolutePath); // Handle root if (absolutePath.isRoot()) { throw new AccessControlException("Cannot create " + f); } if(!isRemoteFile(f)){ if (isDirectory(absolutePath)) { throw new FileAlreadyExistsException("Directory already exists: " + f); } // Only canonicalized path/remote files are allowed throw new IOException("Cannot create non-canonical path " + f); } try { RemotePath remotePath = getRemotePath(absolutePath); return getDelegateFileSystem(remotePath.address).create(remotePath.path, permission, overwrite, bufferSize, replication, blockSize, progress); } catch (IllegalArgumentException e) { throw (IOException) (new IOException("Cannot create file " + absolutePath).initCause(e)); } }