@Override public int run(CommandLine cl) throws AlluxioException, IOException { String[] args = cl.getArgs(); for (String path : args) { AlluxioURI inputPath = new AlluxioURI(path); CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setRecursive(true).build(); mFileSystem.createDirectory(inputPath, options); System.out.println("Successfully created directory " + inputPath); } return 0; }
/** * Attempts to create a folder with the specified path. Parent directories will be created. * * @param path path to create * @param permission permissions to grant the created folder * @return true if the indicated folder is created successfully or already exists */ @Override public boolean mkdirs(Path path, FsPermission permission) throws IOException { LOG.debug("mkdirs({}, {})", path, permission); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setRecursive(true) .setAllowExists(true).setMode(new Mode(permission.toShort()).toProto()).build(); try { mFileSystem.createDirectory(uri, options); return true; } catch (AlluxioException e) { throw new IOException(e); } }
/** * Tests that file information is still present after it has been freed after the parent * directory's TTL has been set to 0. */ @Test public void ttlDirectoryFree() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted because of a TTL of 0. */ @Test public void setTtlForDirectoryWithNoTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); mFileSystemMaster.createDirectory(NESTED_DIR_URI, directoryContext); CreateFileContext createFileContext = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, createFileContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); // Set ttl. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file and directory should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_DIR_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT); }
@Test public void createPathWithWhiteSpaces() throws Exception { String[] paths = new String[]{ "/ ", "/ ", "/ path", "/path ", "/pa th", "/ pa th ", "/pa/ th", "/pa / th", "/ pa / th ", }; for (String path : paths) { AlluxioURI uri = new AlluxioURI(path); long id = mFileSystemMaster.createFile(uri, CreateFileContext.defaults(CreateFilePOptions.newBuilder().setRecursive(true))); Assert.assertEquals(id, mFileSystemMaster.getFileId(uri)); mFileSystemMaster.delete(uri, DeleteContext.defaults()); id = mFileSystemMaster.createDirectory(uri, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); Assert.assertEquals(id, mFileSystemMaster.getFileId(uri)); } }
/** * Tests that TTL free of a directory is not forgotten across restarts. */ @Test public void ttlDirectoryFreeReplay() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); // Simulate restart. stopServices(); startServices(); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
@Test public void removeExtendedDefaultAclMask() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); AclEntry newAcl = AclEntry.fromCliString("default:user:newuser:rwx"); // Add an ACL addAcl(NESTED_URI, newAcl); assertThat(getInfo(NESTED_URI).getDefaultAcl().getEntries(), hasItem(newAcl)); // Attempt to remove the ACL mask AclEntry maskEntry = AclEntry.fromCliString("default:mask::rwx"); assertThat(getInfo(NESTED_URI).getDefaultAcl().getEntries(), hasItem(maskEntry)); try { removeAcl(NESTED_URI, maskEntry); fail("Expected removing the mask from an extended ACL to fail"); } catch (IOException e) { assertThat(e.getMessage(), containsString("mask")); } // Remove the extended ACL removeAcl(NESTED_URI, newAcl); // Now we can add and remove a mask addAcl(NESTED_URI, maskEntry); removeAcl(NESTED_URI, maskEntry); }
@Test public void removeExtendedAclMask() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); AclEntry newAcl = AclEntry.fromCliString("user:newuser:rwx"); // Add an ACL addAcl(NESTED_URI, newAcl); assertThat(getInfo(NESTED_URI).getAcl().getEntries(), hasItem(newAcl)); // Attempt to remove the ACL mask AclEntry maskEntry = AclEntry.fromCliString("mask::rwx"); assertThat(getInfo(NESTED_URI).getAcl().getEntries(), hasItem(maskEntry)); try { removeAcl(NESTED_URI, maskEntry); fail("Expected removing the mask from an extended ACL to fail"); } catch (IOException e) { assertThat(e.getMessage(), containsString("mask")); } // Remove the extended ACL removeAcl(NESTED_URI, newAcl); // Now we can add and remove a mask addAcl(NESTED_URI, maskEntry); removeAcl(NESTED_URI, maskEntry); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for * a directory. */ @Test public void setNoTtlForDirectoryWithTtl() throws Exception { mFileSystemMaster.createDirectory(new AlluxioURI("/nested"), CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setRecursive(true))); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(NESTED_URI.getName(), mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName()); }
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForDirectoryWithTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertTrue( mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName() != null); mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); }
/** * Sets up dependencies before a single test runs. */ @BeforeClass public static void beforeClass() throws Exception { sFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sDirectoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedDirectoryContext = CreateDirectoryContext.defaults( CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); }
public Builder mergeFrom(alluxio.grpc.CreateDirectoryPOptions other) { if (other == alluxio.grpc.CreateDirectoryPOptions.getDefaultInstance()) return this; if (other.hasRecursive()) { setRecursive(other.getRecursive()); } if (other.hasAllowExists()) { setAllowExists(other.getAllowExists()); } if (other.hasMode()) { mergeMode(other.getMode()); } if (other.hasWriteType()) { setWriteType(other.getWriteType()); } if (other.hasCommonOptions()) { mergeCommonOptions(other.getCommonOptions()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
/** * Tests that a file has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the file should not be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for a file. */ @Test public void setNoTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that TTL delete of a directory is not forgotten across restarts. */ @Test public void ttlDirectoryDeleteReplay() throws Exception { CreateDirectoryContext context = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0))); long dirId = mFileSystemMaster.createDirectory(NESTED_DIR_URI, context); // Simulate restart. stopServices(); startServices(); FileInfo fileInfo = mFileSystemMaster.getFileInfo(dirId); assertEquals(fileInfo.getFileId(), dirId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(dirId); }
/** * Tests that a directory has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForDirectoryWithTtl() throws Exception { mFileSystemMaster.createDirectory(new AlluxioURI("/nested"), CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setRecursive(true))); mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the directory should not be deleted during last TTL check. assertEquals(NESTED_URI.getName(), mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName()); }
/** * Tests that an exception is in the * {@literal FileSystemMaster#createDirectory(AlluxioURI, CreateDirectoryOptions)} * with a TTL set in the {@link CreateDirectoryContext} after the TTL check was done once. */ @Test public void ttlDirectoryDelete() throws Exception { CreateDirectoryContext context = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0))); long dirId = mFileSystemMaster.createDirectory(NESTED_DIR_URI, context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(dirId); assertEquals(fileInfo.getFileId(), dirId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(dirId); }
private void verifyCreateDirectory(TestUser user, String path, boolean recursive) throws Exception { try (Closeable r = new AuthenticatedUserRule(user.getUser(), ServerConfiguration.global()).toResource()) { CreateDirectoryContext context = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(recursive)) .setOwner(SecurityUtils.getOwnerFromGrpcClient(ServerConfiguration.global())) .setGroup(SecurityUtils.getGroupFromGrpcClient(ServerConfiguration.global())); mFileSystemMaster.createDirectory(new AlluxioURI(path), context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(mFileSystemMaster.getFileId(new AlluxioURI(path))); String[] pathComponents = path.split("/"); assertEquals(pathComponents[pathComponents.length - 1], fileInfo.getName()); assertEquals(true, fileInfo.isFolder()); assertEquals(user.getUser(), fileInfo.getOwner()); } }
/** * @return Master side defaults for {@link CreateDirectoryPOptions} */ public static CreateDirectoryPOptions createDirectoryDefaults() { return CreateDirectoryPOptions.newBuilder() .setCommonOptions(commonDefaults()) .setMode(ModeUtils.applyDirectoryUMask(Mode.defaults(), ServerConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)).toProto()) .setRecursive(false) .setWriteType(ServerConfiguration .getEnum(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class).toProto()) .setAllowExists(false).build(); }
@Override public void apply(FileSystem fs) throws Exception { Utils.createFile(fs, SRC); fs.rename(SRC, DST); fs.createDirectory(new AlluxioURI("/renameDir/a/b"), CreateDirectoryPOptions.newBuilder().setRecursive(true).build()); fs.rename(new AlluxioURI("/renameDir/a"), new AlluxioURI("/renameDir/c")); }