/** * Tests that the async write invokes the expected client APIs. */ @Test public void asyncWrite() throws Exception { OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setWriteType(WriteType.ASYNC_THROUGH); mTestStream = createTestStream(FILE_NAME, options); mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.close(); verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); verify(mFileSystemMasterClient).scheduleAsyncPersist(eq(FILE_NAME)); }
/** * Tests for the {@link BaseFileSystem#createDirectory(AlluxioURI, CreateDirectoryPOptions)} * method. */ @Test public void createDirectory() throws Exception { AlluxioURI dir = new AlluxioURI("/dir"); CreateDirectoryPOptions createDirectoryOptions = CreateDirectoryPOptions.getDefaultInstance(); doNothing().when(mFileSystemMasterClient).createDirectory(dir, createDirectoryOptions); mFileSystem.createDirectory(dir, createDirectoryOptions); verify(mFileSystemMasterClient).createDirectory(dir, createDirectoryOptions); verifyFilesystemContextAcquiredAndReleased(); }
/** * Tests the creation of a file via the * {@link BaseFileSystem#createFile(AlluxioURI, CreateFilePOptions)} method. */ @Test public void createFile() throws Exception { doNothing().when(mFileSystemMasterClient) .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class)); URIStatus status = new URIStatus(new FileInfo()); AlluxioURI file = new AlluxioURI("/file"); GetStatusPOptions getStatusOptions = GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).build(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); FileOutStream out = mFileSystem.createFile(file, CreateFilePOptions.getDefaultInstance()); verify(mFileSystemMasterClient).createFile(file, CreateFilePOptions.getDefaultInstance()); assertEquals(out.mUri, file); verifyFilesystemContextAcquiredAndReleased(); }
@Override public boolean exists(AlluxioURI path, ExistsPOptions options) throws InvalidPathException, IOException, AlluxioException { checkUri(path); FileSystemMasterClient masterClient = mFsContext.acquireMasterClient(); try { // TODO(calvin): Make this more efficient masterClient.getStatus(path, GrpcUtils.toGetStatusOptions(options)); return true; } catch (NotFoundException e) { return false; } catch (InvalidArgumentException e) { // The server will throw this when a prefix of the path is a file. // TODO(andrew): Change the server so that a prefix being a file means the path does not exist return false; } catch (UnavailableException e) { throw e; } catch (AlluxioStatusException e) { throw e.toAlluxioException(); } finally { mFsContext.releaseMasterClient(masterClient); } }
/** * Schedules the async persistence of the current file. */ protected void scheduleAsyncPersist() throws IOException { try (CloseableResource<FileSystemMasterClient> masterClient = mContext .acquireMasterClientResource()) { masterClient.get().scheduleAsyncPersist(mUri); } }
when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class), any(GetStatusPOptions.class))) .thenReturn(new URIStatus(new FileInfo())); when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)) .thenAnswer(new Answer<Long>() { private long mCount = 0;
@Override public void delete(AlluxioURI path, DeletePOptions options) throws DirectoryNotEmptyException, FileDoesNotExistException, IOException, AlluxioException { checkUri(path); FileSystemMasterClient masterClient = mFsContext.acquireMasterClient(); try { masterClient.delete(path, options); LOG.debug("Deleted {}, options: {}", path.getPath(), options); } catch (FailedPreconditionException e) { // A little sketchy, but this should be the only case that throws FailedPrecondition. throw new DirectoryNotEmptyException(e.getMessage()); } catch (NotFoundException e) { throw new FileDoesNotExistException(e.getMessage()); } catch (UnavailableException e) { throw e; } catch (AlluxioStatusException e) { throw e.toAlluxioException(); } finally { mFsContext.releaseMasterClient(masterClient); } }
@Override public void free(AlluxioURI path, FreePOptions options) throws FileDoesNotExistException, IOException, AlluxioException { checkUri(path); FileSystemMasterClient masterClient = mFsContext.acquireMasterClient(); try { masterClient.free(path, options); LOG.debug("Freed {}, options: {}", path.getPath(), options); } catch (NotFoundException e) { throw new FileDoesNotExistException(e.getMessage()); } catch (UnavailableException e) { throw e; } catch (AlluxioStatusException e) { throw e.toAlluxioException(); } finally { mFsContext.releaseMasterClient(masterClient); } }
/** * Tests for the {@link BaseFileSystem#listStatus(AlluxioURI, ListStatusPOptions)} method. */ @Test public void listStatus() throws Exception { AlluxioURI file = new AlluxioURI("/file"); List<URIStatus> infos = new ArrayList<>(); infos.add(new URIStatus(new FileInfo())); ListStatusPOptions listStatusOptions = ListStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.listStatus(file, listStatusOptions)).thenReturn(infos); assertSame(infos, mFileSystem.listStatus(file, listStatusOptions)); verify(mFileSystemMasterClient).listStatus(file, listStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }
/** * Tests that {@link FileOutStream#close()} will close but not cancel the underlying out streams. * Also checks that {@link FileOutStream#close()} persists and completes the file. */ @Test public void close() throws Exception { mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.close(); for (long streamIndex = 0; streamIndex < 2; streamIndex++) { Assert.assertFalse(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); } verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); }
/** * Runs report ufs command. * * @return 0 on success, 1 otherwise */ public int run() throws IOException { Map<String, MountPointInfo> mountTable = mFileSystemMasterClient.getMountTable(); System.out.println("Alluxio under storage system information:"); printMountInfo(mountTable); return 0; }
private long getNextBlockId() throws IOException { try (CloseableResource<FileSystemMasterClient> masterClient = mContext .acquireMasterClientResource()) { return masterClient.get().getNewBlockIdForFile(mUri); } }
/** * Ensures that an exception is propagated correctly when creating a file system. */ @Test public void createException() throws Exception { doThrow(EXCEPTION).when(mFileSystemMasterClient) .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class)); try { mFileSystem.createFile(new AlluxioURI("/"), CreateFilePOptions.getDefaultInstance()); fail(SHOULD_HAVE_PROPAGATED_MESSAGE); } catch (Exception e) { assertSame(EXCEPTION, e); } verifyFilesystemContextAcquiredAndReleased(); }
@Override public URIStatus getStatus(AlluxioURI path, GetStatusPOptions options) throws FileDoesNotExistException, IOException, AlluxioException { checkUri(path); FileSystemMasterClient masterClient = mFsContext.acquireMasterClient(); try { return masterClient.getStatus(path, options); } catch (NotFoundException e) { throw new FileDoesNotExistException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path)); } catch (UnavailableException e) { throw e; } catch (AlluxioStatusException e) { throw e.toAlluxioException(); } finally { mFsContext.releaseMasterClient(masterClient); } }
/** * Persists the given file to the under file system. * * @param fs {@link FileSystem} to carry out Alluxio operations * @param fsContext the {@link FileSystemContext} linked to the {@link FileSystem} client * @param uri the uri of the file to persist */ public static void persistFile(final FileSystem fs, final FileSystemContext fsContext, final AlluxioURI uri) throws IOException, TimeoutException, InterruptedException { FileSystemMasterClient client = fsContext.acquireMasterClient(); try { client.scheduleAsyncPersist(uri); } finally { fsContext.releaseMasterClient(client); } CommonUtils.waitFor(String.format("%s to be persisted", uri) , () -> { try { return fs.getStatus(uri).isPersisted(); } catch (Exception e) { Throwables.propagateIfPossible(e); throw new RuntimeException(e); } }, WaitForOptions.defaults().setTimeoutMs(20 * Constants.MINUTE_MS) .setInterval(Constants.SECOND_MS)); }
/** * Tests for the {@link BaseFileSystem#delete(AlluxioURI, DeletePOptions)} method. */ @Test public void delete() throws Exception { AlluxioURI file = new AlluxioURI("/file"); DeletePOptions deleteOptions = DeletePOptions.newBuilder().setRecursive(true).build(); mFileSystem.delete(file, deleteOptions); verify(mFileSystemMasterClient).delete(file, deleteOptions); verifyFilesystemContextAcquiredAndReleased(); }
/** * Tests for the {@link BaseFileSystem#free(AlluxioURI, FreePOptions)} method. */ @Test public void free() throws Exception { AlluxioURI file = new AlluxioURI("/file"); FreePOptions freeOptions = FreePOptions.newBuilder().setRecursive(true).build(); mFileSystem.free(file, freeOptions); verify(mFileSystemMasterClient).free(file, freeOptions); verifyFilesystemContextAcquiredAndReleased(); }
@Override public List<URIStatus> listStatus(AlluxioURI path, ListStatusPOptions options) throws FileDoesNotExistException, IOException, AlluxioException { checkUri(path); FileSystemMasterClient masterClient = mFsContext.acquireMasterClient(); // TODO(calvin): Fix the exception handling in the master try { return masterClient.listStatus(path, options); } catch (NotFoundException e) { throw new FileDoesNotExistException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path)); } catch (UnavailableException e) { throw e; } catch (AlluxioStatusException e) { throw e.toAlluxioException(); } finally { mFsContext.releaseMasterClient(masterClient); } }
/** * Tests that {@link FileOutStream#cancel()} will cancel and close the underlying out streams, and * delete from the under file system when the delegation flag is set. Also makes sure that * cancel() doesn't persist or complete the file. */ @Test public void cancelWithDelegation() throws Exception { mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.cancel(); for (long streamIndex = 0; streamIndex < 2; streamIndex++) { Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); } // Don't complete the file if the stream was canceled verify(mFileSystemMasterClient, times(0)).completeFile(any(AlluxioURI.class), any(CompleteFilePOptions.class)); }
@Override public void check(Clients clients) throws Exception { FileSystemMasterClient masterClient = clients.getFileSystemMasterClient(); assertFalse(masterClient.getMountTable().get("/").getReadOnly()); } }