private void writeFile(FileSystem fs) throws IOException, AlluxioException { for (int i = 0; i < mNumFiles; i++) { ByteBuffer buf = ByteBuffer.allocate(80); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { buf.putInt(k); } buf.flip(); AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Writing data to {}", filePath); OutputStream os = fs.createFile(filePath, CreateFilePOptions.newBuilder().setRecursive(true).build()); os.write(buf.array()); os.close(); } }
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { LOG.debug("append({}, {}, {})", path, bufferSize, progress); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); try { if (mFileSystem.exists(uri)) { throw new IOException(ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(uri)); } return new FSDataOutputStream( mFileSystem.createFile(uri, CreateFilePOptions.newBuilder().setRecursive(true).build()), mStatistics); } catch (AlluxioException e) { throw new IOException(e); } }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path */ public static void createFile(FileSystem fs, AlluxioURI path) throws Exception { try (FileOutStream out = fs.createFile(path, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setRecursive(true).build())) { out.write("test".getBytes()); } }
/** * Sets up dependencies before a single test runs. */ @BeforeClass public static void beforeClass() throws Exception { sFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sDirectoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedDirectoryContext = CreateDirectoryContext.defaults( CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForFileWithTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since TTL is 1 hour, the file won't be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that a file has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the file should not be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that TTL delete of a file is not forgotten across restarts. */ @Test public void ttlFileDeleteReplay() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // Simulate restart. stopServices(); startServices(); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for a file. */ @Test public void setNoTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that an exception is in the * {@link FileSystemMaster#createFile(AlluxioURI, CreateFileContext)} with a * TTL set in the {@link CreateFileContext} after the TTL check was done once. */ @Test public void ttlFileDelete() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
/** * @return Master side defaults for {@link CreateFilePOptions} */ public static CreateFilePOptions createFileDefaults() { return CreateFilePOptions.newBuilder() .setCommonOptions(commonDefaults()) .setRecursive(false) .setBlockSizeBytes(ServerConfiguration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT)) .setFileWriteLocationPolicy( ServerConfiguration.get(PropertyKey.USER_FILE_WRITE_LOCATION_POLICY)) .setWriteTier(ServerConfiguration.getInt(PropertyKey.USER_FILE_WRITE_TIER_DEFAULT)) .setWriteType(ServerConfiguration .getEnum(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class).toProto()) .setMode(ModeUtils.applyFileUMask(Mode.defaults(), ServerConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)).toProto()) .setReplicationDurable(ServerConfiguration .getInt(PropertyKey.USER_FILE_REPLICATION_DURABLE)) .setReplicationMin(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MIN)) .setReplicationMax(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MAX)) .build(); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted because of a TTL of 0. */ @Test public void setTtlForFileWithNoTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
private void verifyCreateFile(TestUser user, String path, boolean recursive) throws Exception { try (Closeable r = new AuthenticatedUserRule(user.getUser(), ServerConfiguration.global()).toResource()) { CreateFileContext context = CreateFileContext .defaults( CreateFilePOptions.newBuilder().setRecursive(recursive)) .setOwner(SecurityUtils.getOwnerFromGrpcClient(ServerConfiguration.global())) .setGroup(SecurityUtils.getGroupFromGrpcClient(ServerConfiguration.global())) .setPersisted(true); long fileId = mFileSystemMaster.createFile(new AlluxioURI(path), context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); String[] pathComponents = path.split("/"); assertEquals(pathComponents[pathComponents.length - 1], fileInfo.getName()); assertEquals(user.getUser(), fileInfo.getOwner()); } }
/** * Sets up the dependencies before a test runs. */ @Before public void before() throws Exception { GroupMappingServiceTestUtils.resetCache(); MetricsSystem.clearAllMetrics(); // This makes sure that the mount point of the UFS corresponding to the Alluxio root ("/") // doesn't exist by default (helps loadRootTest). mUnderFS = ServerConfiguration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); mNestedFileContext = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); mJournalFolder = mTestFolder.newFolder().getAbsolutePath(); startServices(); }
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileName the name of the file to be created * @param writeType {@link WritePType} used to create the file * @param len file size * @param blockCapacityByte block size of the file */ public static void createByteFile(FileSystem fs, String fileName, WritePType writeType, int len, long blockCapacityByte) { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(writeType) .setBlockSizeBytes(blockCapacityByte).setRecursive(true).build(); createByteFile(fs, new AlluxioURI(fileName), options, len); }
private FileOutStream createFile(FileSystem fileSystem, AlluxioURI filePath, boolean deleteIfExists) throws IOException, AlluxioException { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(mWriteType.toProto()) .setRecursive(true).build(); if (!fileSystem.exists(filePath)) { // file doesn't exist yet, so create it return fileSystem.createFile(filePath, options); } else if (deleteIfExists) { // file exists, so delete it and recreate fileSystem.delete(filePath); return fileSystem.createFile(filePath, options); } // file exists and deleteIfExists is false throw new FileAlreadyExistsException("File exists and deleteIfExists is false"); }
/** * @param filePath the path for the files * @param readType the {@link ReadPType} * @param writeType the {@link WritePType} * @param fsContext the {@link FileSystemContext } to use for client operations */ public BasicOperations(AlluxioURI filePath, ReadType readType, WriteType writeType, FileSystemContext fsContext) { mFilePath = filePath; mReadOptions = OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build(); mWriteOptions = CreateFilePOptions.newBuilder().setWriteType(writeType.toProto()) .setRecursive(true).build(); mFsContext = fsContext; }
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileURI URI of the file * @param writeType {@link WritePType} used to create the file * @param len file size */ public static void createByteFile(FileSystem fs, AlluxioURI fileURI, WritePType writeType, int len) { CreateFilePOptions options = CreateFilePOptions.newBuilder().setRecursive(true).setWriteType(writeType).build(); createByteFile(fs, fileURI, options, len); }
@Override public void apply(FileSystem fs) throws Exception { Utils.createFile(fs, PATH); Utils.createFile(fs, NESTED); Utils.createFile(fs, MODE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setMode(TEST_MODE.toProto()).build()); Utils.createFile(fs, THROUGH, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setWriteType(WritePType.THROUGH).build()); Utils.createFile(fs, TTL, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(TEST_TTL) .setTtlAction(alluxio.grpc.TtlAction.FREE)) .build()); }
@Test public void renameToNonExistentParent() throws Exception { CreateFileContext context = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); mFileSystemMaster.createFile(NESTED_URI, context); try { mFileSystemMaster.rename(NESTED_URI, new AlluxioURI("/testDNE/b"), RenameContext.defaults()); fail("Rename to a non-existent parent path should not succeed."); } catch (FileDoesNotExistException e) { // Expected case. } }