@Override public void check(FileSystem fs) throws Exception { for (AlluxioURI deleted : Arrays.asList(PATH, NESTED, RECURSIVE)) { assertFalse(fs.exists(deleted)); } assertTrue(fs.exists(NESTED.getParent())); assertFalse(fs.exists(RECURSIVE.getParent())); assertTrue(fs.exists(RECURSIVE.getParent().getParent())); } }
@Override public void check(Clients clients) throws Exception { for (AlluxioURI uri : Arrays.asList(PATH, NESTED_PATH, NESTED_PATH.getParent())) { assertTrue(clients.getFileSystemClient().getStatus(uri).isPersisted()); } } }
private void checkNestedBucketIsUnderMountPoint(String bucketPath) throws S3Exception { // Assure that the bucket is directly under a mount point. AlluxioURI parent = new AlluxioURI(bucketPath).getParent(); try { if (!mFileSystem.getMountTable().containsKey(parent.getPath())) { throw new S3Exception(bucketPath, S3ErrorCode.INVALID_NESTED_BUCKET_NAME); } } catch (Exception e) { throw toBucketS3Exception(e, bucketPath); } }
/** * Gets all the {@link AlluxioURI}s that match inputURI. If the path is a regular path, the * returned list only contains the corresponding URI; Else if the path contains wildcards, the * returned list contains all the matched URIs It supports any number of wildcards in inputURI * * @param alluxioClient the client used to fetch information of Alluxio files * @param inputURI the input URI (could contain wildcards) * @return a list of {@link AlluxioURI}s that matches the inputURI */ public static List<AlluxioURI> getAlluxioURIs(FileSystem alluxioClient, AlluxioURI inputURI) throws IOException { if (!inputURI.getPath().contains(AlluxioURI.WILDCARD)) { return Lists.newArrayList(inputURI); } else { String inputPath = inputURI.getPath(); AlluxioURI parentURI = new AlluxioURI(inputURI.getScheme(), inputURI.getAuthority(), inputPath.substring(0, inputPath.indexOf(AlluxioURI.WILDCARD) + 1), inputURI.getQueryMap()).getParent(); return getAlluxioURIs(alluxioClient, inputURI, parentURI); } }
@Override public boolean isAbsent(AlluxioURI path) { MountInfo mountInfo = getMountInfo(path); if (mountInfo == null) { return false; } AlluxioURI mountBaseUri = mountInfo.getAlluxioUri(); while (path != null && !path.equals(mountBaseUri)) { Long cached = mCache.getIfPresent(path.getPath()); if (cached != null && cached == mountInfo.getMountId()) { return true; } path = path.getParent(); } // Reached the root, without finding anything in the cache. return false; }
/** * @param rpcContext the rpc context * @param inodePath the path to be created * @param context the method context * @return the list of created inodes */ List<Inode> createFileInternal(RpcContext rpcContext, LockedInodePath inodePath, CreateFileContext context) throws InvalidPathException, FileAlreadyExistsException, BlockInfoException, IOException, FileDoesNotExistException { if (mWhitelist.inList(inodePath.getUri().toString())) { context.setCacheable(true); } // If the create succeeded, the list of created inodes will not be empty. List<Inode> created = mInodeTree.createPath(rpcContext, inodePath, context); if (context.isPersisted()) { // The path exists in UFS, so it is no longer absent. The ancestors exist in UFS, but the // actual file does not exist in UFS yet. mUfsAbsentPathCache.processExisting(inodePath.getUri().getParent()); } Metrics.FILES_CREATED.inc(); Metrics.DIRECTORIES_CREATED.inc(); return created; }
/** * @return the parent of the target inode * @throws InvalidPathException if the parent inode is not a directory * @throws FileDoesNotExistException if the parent of the target does not exist */ public InodeDirectory getParentInodeDirectory() throws InvalidPathException, FileDoesNotExistException { Inode inode = getParentInodeOrNull(); if (inode == null) { throw new FileDoesNotExistException( ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(mUri.getParent())); } if (!inode.isDirectory()) { throw new InvalidPathException( ExceptionMessage.PATH_MUST_HAVE_VALID_PARENT.getMessage(mUri)); } return (InodeDirectory) inode; }
/** * Tests the {@link AlluxioURI#getParent()} method. */ @Test public void getParentTests() { assertEquals(null, new AlluxioURI("/").getParent()); assertEquals(null, new AlluxioURI("alluxio://localhost/").getParent()); assertEquals(new AlluxioURI("alluxio://localhost/"), new AlluxioURI("alluxio://localhost/a").getParent()); assertEquals(new AlluxioURI("/a"), new AlluxioURI("/a/b/../c").getParent()); assertEquals(new AlluxioURI("alluxio:/a"), new AlluxioURI("alluxio:/a/b/../c").getParent()); assertEquals(new AlluxioURI("alluxio://localhost:80/a"), new AlluxioURI( "alluxio://localhost:80/a/b/../c").getParent()); }
@Override public void check(FileSystem fs) throws Exception { assertTrue(fs.getStatus(FILE).isPersisted()); assertTrue(fs.getStatus(NESTED_FILE).isPersisted()); assertTrue(fs.getStatus(NESTED_FILE.getParent()).isPersisted()); } }
@Override public void apply(FileSystem fs) throws Exception { Utils.createFile(fs, PATH); fs.delete(PATH); Utils.createFile(fs, NESTED); fs.delete(NESTED); Utils.createFile(fs, RECURSIVE); fs.delete(RECURSIVE.getParent(), DeletePOptions.newBuilder().setRecursive(true).build()); }
/** * Checks for absent paths (descendants) and existing paths (ancestors) in the UFS. * * @param firstAbsent the first Alluxio path which should not exist in the UFS */ private void checkAbsentPaths(AlluxioURI firstAbsent) throws Exception { // Check for additional non-existing paths as descendants of the first absent path for (int level = 1; level <= 2; level++) { AlluxioURI levelUri = firstAbsent.join("level" + level); for (int dir = 1; dir <= 2; dir++) { AlluxioURI uri = levelUri.join("dir" + dir); Assert.assertTrue(uri.toString(), mUfsAbsentPathCache.isAbsent(uri)); } } // Check all ancestors AlluxioURI existing = firstAbsent.getParent(); while (existing != null) { Assert.assertFalse(existing.toString(), mUfsAbsentPathCache.isAbsent(existing)); existing = existing.getParent(); } } }
/** * Tests the {@link FileSystemMaster#free} method with a directory. */ @Test public void freeDir() throws Exception { mNestedFileContext.setPersisted(true); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // free the dir mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setForced(true).setRecursive(true))); // Update the heartbeat of removedBlockId received from worker 1. Command heartbeat3 = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat3); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests mounting a prefix UFS dir. */ @Test public void mountPrefixUfsDir() throws Exception { AlluxioURI ufsURI = createTempUfsDir("ufs/hello/shadow"); AlluxioURI alluxioURI = new AlluxioURI("/hello"); mFileSystemMaster.mount(alluxioURI, ufsURI, MountContext.defaults()); AlluxioURI preUfsURI = ufsURI.getParent(); AlluxioURI anotherAlluxioURI = new AlluxioURI("/hi"); mThrown.expect(InvalidPathException.class); mFileSystemMaster.mount(anotherAlluxioURI, preUfsURI, MountContext.defaults()); }
/** * Tests mounting a suffix UFS dir. */ @Test public void mountSuffixUfsDir() throws Exception { AlluxioURI ufsURI = createTempUfsDir("ufs/hello/shadow"); AlluxioURI preUfsURI = ufsURI.getParent(); AlluxioURI alluxioURI = new AlluxioURI("/hello"); mFileSystemMaster.mount(alluxioURI, preUfsURI, MountContext.defaults()); AlluxioURI anotherAlluxioURI = new AlluxioURI("/hi"); mThrown.expect(InvalidPathException.class); mFileSystemMaster.mount(anotherAlluxioURI, ufsURI, MountContext.defaults()); }
/** * Tests the {@link FileSystemMaster#free} method with a directory with a file pinned when * forced flag is true. */ @Test public void freeDirWithPinnedFileAndForced() throws Exception { mNestedFileContext.setPersisted(true); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setPinned(true))); // free the parent dir of a pinned file with "forced" mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setForced(true).setRecursive(true))); // Update the heartbeat of removedBlockId received from worker 1. Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests the {@link FileSystemMaster#free} method with a directory but recursive to false. */ @Test public void freeDirNonRecursive() throws Exception { mNestedFileContext.setPersisted(true); createFileWithSingleBlock(NESTED_FILE_URI); mThrown.expect(UnexpectedAlluxioException.class); mThrown.expectMessage(ExceptionMessage.CANNOT_FREE_NON_EMPTY_DIR.getMessage(NESTED_URI)); // cannot free directory with recursive argument to false mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setRecursive(false))); }
/** * Tests the {@link FileSystemMaster#free} method with a directory with a file non-persisted. */ @Test public void freeDirWithNonPersistedFile() throws Exception { createFileWithSingleBlock(NESTED_FILE_URI); mThrown.expect(UnexpectedAlluxioException.class); mThrown.expectMessage(ExceptionMessage.CANNOT_FREE_NON_PERSISTED_FILE .getMessage(NESTED_FILE_URI.getPath())); // cannot free the parent dir of a non-persisted file mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setForced(false).setRecursive(true))); }
@Test public void basicMultiMasterUri() { AlluxioURI uri = new AlluxioURI("alluxio://host1:19998,host2:19998,host3:19998/xy z/a b c"); assertTrue(uri.hasAuthority()); assertEquals("host1:19998,host2:19998,host3:19998", uri.getAuthority().toString()); assertTrue(uri.getAuthority() instanceof MultiMasterAuthority); assertEquals(2, uri.getDepth()); assertEquals("a b c", uri.getName()); assertEquals("alluxio://host1:19998,host2:19998,host3:19998/xy z", uri.getParent().toString()); assertEquals("alluxio://host1:19998,host2:19998,host3:19998/", uri.getParent().getParent().toString()); assertEquals("/xy z/a b c", uri.getPath()); assertEquals("alluxio", uri.getScheme()); assertTrue(uri.hasScheme()); assertTrue(uri.isAbsolute()); assertTrue(uri.isPathAbsolute()); assertEquals("alluxio://host1:19998,host2:19998,host3:19998/xy z/a b c/d", uri.join("/d").toString()); assertEquals("alluxio://host1:19998,host2:19998,host3:19998/xy z/a b c/d", uri.join(new AlluxioURI("/d")).toString()); assertEquals("alluxio://host1:19998,host2:19998,host3:19998/xy z/a b c", uri.toString()); }
/** * Tests mounting a shadow Alluxio dir. */ @Test public void mountShadowDir() throws Exception { AlluxioURI alluxioURI = new AlluxioURI("/hello"); AlluxioURI ufsURI = createTempUfsDir("ufs/hello/shadow"); mFileSystemMaster.mount(alluxioURI, ufsURI.getParent(), MountContext.defaults()); AlluxioURI shadowAlluxioURI = new AlluxioURI("/hello/shadow"); AlluxioURI notShadowAlluxioURI = new AlluxioURI("/hello/notshadow"); AlluxioURI shadowUfsURI = createTempUfsDir("ufs/hi"); AlluxioURI notShadowUfsURI = createTempUfsDir("ufs/notshadowhi"); mFileSystemMaster.mount(notShadowAlluxioURI, notShadowUfsURI, MountContext.defaults()); mThrown.expect(IOException.class); mFileSystemMaster.mount(shadowAlluxioURI, shadowUfsURI, MountContext.defaults()); }
/** * Tests the {@link FileSystemMaster#free} method with a directory with a file pinned when * forced flag is false. */ @Test public void freeDirWithPinnedFileAndNotForced() throws Exception { mNestedFileContext.setPersisted(true); createFileWithSingleBlock(NESTED_FILE_URI); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setPinned(true))); mThrown.expect(UnexpectedAlluxioException.class); mThrown.expectMessage(ExceptionMessage.CANNOT_FREE_PINNED_FILE .getMessage(NESTED_FILE_URI.getPath())); // cannot free the parent dir of a pinned file without "forced" mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setForced(false).setRecursive(true))); }