/** * Creates a {@link FileSystemContext} with a null subject. * * @param alluxioConf Alluxio configuration * @return an instance of file system context with no subject associated */ public static FileSystemContext create(AlluxioConfiguration alluxioConf) { return create(null, alluxioConf); }
@Override public long getDefaultBlockSize() { return mFsContext.getConf() .getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT); }
/** * Checks the consistency of Alluxio metadata against the under storage for all files and * directories in a given subtree. * * @param fsContext the {@link FileSystemContext} linked to the {@link FileSystem} client * @param path the root of the subtree to check * @param options method options * @return a list of inconsistent files and directories */ public static List<AlluxioURI> checkConsistency(FileSystemContext fsContext, AlluxioURI path, CheckConsistencyPOptions options) throws IOException { FileSystemMasterClient client = fsContext.acquireMasterClient(); try { return client.checkConsistency(path, options); } finally { fsContext.releaseMasterClient(client); } } }
/** * @param context the file system context * @param address the data server address * @param id the block or UFS ID * @param length the length of the block or file to write, set to Long.MAX_VALUE if unknown * @param type type of the write request * @param options the options of the output stream * @return an instance of {@link GrpcDataWriter} */ public static GrpcDataWriter create(FileSystemContext context, WorkerNetAddress address, long id, long length, RequestType type, OutStreamOptions options) throws IOException { AlluxioConfiguration conf = context.getConf(); long chunkSize = conf.getBytes(PropertyKey.USER_NETWORK_WRITER_CHUNK_SIZE_BYTES); BlockWorkerClient grpcClient = context.acquireBlockWorkerClient(address); try { return new GrpcDataWriter(context, address, id, length, chunkSize, type, options, grpcClient); } catch (Exception e) { context.releaseBlockWorkerClient(address, grpcClient); throw e; } }
@Override public int run(CommandLine cl) { try (CloseableResource<FileSystemMasterClient> client = mFsContext.acquireMasterClientResource()) { try { InetSocketAddress address = client.get().getAddress(); System.out.println(address.getHostName()); List<InetSocketAddress> addresses = Arrays.asList(address); MasterInquireClient inquireClient = new PollingMasterInquireClient(addresses, () -> new ExponentialBackoffRetry(50, 100, 2), mFsContext.getConf() ); try { inquireClient.getPrimaryRpcAddress(); } catch (UnavailableException e) { System.err.println("The leader is not currently serving requests."); } } catch (UnavailableException e) { System.err.println("Failed to get the leader master."); } } return 0; }
@Before public void before() throws Exception { mClientContext = ClientContext.create(mConf); mContext = PowerMockito.mock(FileSystemContext.class); mAddress = Mockito.mock(WorkerNetAddress.class); mClient = mock(BlockWorkerClient.class); mRequestObserver = mock(ClientCallStreamObserver.class); PowerMockito.when(mContext.getClientContext()).thenReturn(mClientContext); PowerMockito.when(mContext.getConf()).thenReturn(mConf); PowerMockito.when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn(mClient); PowerMockito.doNothing().when(mContext).releaseBlockWorkerClient(mAddress, mClient); PowerMockito.when(mClient.writeBlock(any(StreamObserver.class))).thenReturn(mRequestObserver); PowerMockito.when(mRequestObserver.isReady()).thenReturn(true); }
FileSystemContext fsContext = FileSystemContext.create( ClientContext.create(mConf)); for (int i = 0; i < mConf.getInt(PropertyKey.USER_FILE_MASTER_CLIENT_THREADS); i++) { clients.add(fsContext.acquireMasterClient()); fsContext.releaseMasterClient(client);
throws FileAlreadyExistsException, InvalidPathException, IOException, AlluxioException { checkUri(path); FileSystemMasterClient masterClient = mFsContext.acquireMasterClient(); URIStatus status; try { throw e.toAlluxioException(); } finally { mFsContext.releaseMasterClient(masterClient); new OutStreamOptions(options, mFsContext.getConf()); outStreamOptions.setUfsPath(status.getUfsPath()); outStreamOptions.setMountId(status.getMountId());
/** * Sets up the file system and the context before a test runs. */ @Before public void before() { mClientContext = ClientContext.create(mConf); mFileContext = PowerMockito.mock(FileSystemContext.class); mFileSystem = new DummyAlluxioFileSystem(mFileContext); mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class); when(mFileContext.acquireMasterClient()).thenReturn(mFileSystemMasterClient); when(mFileContext.getClientContext()).thenReturn(mClientContext); when(mFileContext.getConf()).thenReturn(mConf); }
@Before public void before() throws Exception { mMasterClient = PowerMockito.mock(BlockMasterClient.class); mWorkerClient = PowerMockito.mock(BlockWorkerClient.class); mContext = PowerMockito.mock(FileSystemContext.class); when(mContext.acquireBlockMasterClientResource()) .thenReturn(new DummyCloseableResource<>(mMasterClient)); when(mContext.getClientContext()).thenReturn(ClientContext.create(sConf)); when(mContext.getConf()).thenReturn(sConf); mLocalAddr = new WorkerNetAddress().setHost(NetworkAddressUtils.getLocalHostName( (int) sConf.getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS))); mBlockStore = new AlluxioBlockStore(mContext, TieredIdentityFactory.fromString("node=" + WORKER_HOSTNAME_LOCAL, sConf)); when(mContext.acquireBlockWorkerClient(any(WorkerNetAddress.class))) .thenReturn(mWorkerClient); mStreamObserver = PowerMockito.mock(ClientCallStreamObserver.class); when(mWorkerClient.writeBlock(any(StreamObserver.class))) .thenReturn(mStreamObserver); when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) .thenReturn(mStreamObserver); when(mStreamObserver.isReady()).thenReturn(true); }
/** * Checks if the master client service is available. * Throws an exception if fails to determine that the master client service is running. * * @param alluxioConf Alluxio configuration */ public static void checkMasterClientService(AlluxioConfiguration alluxioConf) throws IOException { try (CloseableResource<FileSystemMasterClient> client = FileSystemContext.create(ClientContext.create(alluxioConf)) .acquireMasterClientResource()) { InetSocketAddress address = client.get().getAddress(); List<InetSocketAddress> addresses = Arrays.asList(address); MasterInquireClient inquireClient = new PollingMasterInquireClient(addresses, () -> new ExponentialBackoffRetry(50, 100, 2), alluxioConf); inquireClient.getPrimaryRpcAddress(); } catch (UnavailableException e) { throw new IOException("Cannot connect to Alluxio leader master."); } }
FileSystemContext.get().reset(Configuration.global()); FileSystemMasterClient client = FileSystemContext.get().acquireMasterClient(); try { client.connect(); FileSystemContext.get().releaseMasterClient(client);
if (cache && (mLastBlockIdCached != blockId)) { WorkerNetAddress worker; if (mPassiveCachingEnabled && mContext.hasLocalWorker()) { // send request to local worker worker = mContext.getLocalWorker(); } else { // send request to data source worker = dataSource; .setSourceHost(dataSource.getHost()).setSourcePort(dataSource.getDataPort()) .build(); BlockWorkerClient blockWorker = mContext.acquireBlockWorkerClient(worker); try { blockWorker.asyncCache(request); mLastBlockIdCached = blockId; } finally { mContext.releaseBlockWorkerClient(worker, blockWorker);
@Before public void before() throws Exception { AlluxioConfiguration conf = ConfigurationTestUtils.defaults(); mMockFileSystem = Mockito.mock(FileSystem.class); mMockFileSystemContext = PowerMockito.mock(FileSystemContext.class); when(mMockFileSystemContext.getClientContext()) .thenReturn(ClientContext.create(conf)); when(mMockFileSystemContext.getConf()) .thenReturn(conf); mMockInStream = new MockFileInStream(mMockFileSystemContext, TEST_SOURCE_CONTENTS, conf); when(mMockFileSystem.openFile(new AlluxioURI(TEST_SOURCE))).thenReturn(mMockInStream); mMockOutStream = new MockFileOutStream(mMockFileSystemContext); when(mMockFileSystem.createFile(eq(new AlluxioURI(TEST_DESTINATION)), any(CreateFilePOptions.class))).thenReturn(mMockOutStream); mMockUfsManager = Mockito.mock(UfsManager.class); }
final WorkerNetAddress address, long blockId, OutStreamOptions options) throws IOException { AlluxioConfiguration conf = context.getConf(); long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES); final BlockWorkerClient blockWorker = context.acquireBlockWorkerClient(address); Closer closer = Closer.create(); try {
BlockWorkerClient blockWorker = null; try { blockWorker = mFsContext.acquireBlockWorkerClient(localNetAddress); blockWorker.removeBlock(request); } catch (NotFoundException e) { } finally { if (blockWorker != null) { mFsContext.releaseBlockWorkerClient(localNetAddress, blockWorker);
/** * Resets the client pools to the original state. */ protected void resetClientPools() throws IOException { Configuration.set(PropertyKey.USER_METRICS_COLLECTION_ENABLED, false); FileSystemContext.clearCache(); FileSystemContext.get().reset(Configuration.global()); }
private long getNextBlockId() throws IOException { try (CloseableResource<FileSystemMasterClient> masterClient = mContext .acquireMasterClientResource()) { return masterClient.get().getNewBlockIdForFile(mUri); } }
@Override public void close() throws IOException { context.releaseBlockWorkerClient(address, blockWorker); } });
/** * Starts both master and a worker using the configurations in test conf respectively. */ public void start() throws Exception { // Disable HDFS client caching to avoid file system close() affecting other clients System.setProperty("fs.hdfs.impl.disable.cache", "true"); resetClientPools(); setupTest(); startMasters(); // Reset the file system context to make sure the correct master RPC port is used. FileSystemContext.get().reset(Configuration.global()); startWorkers(); startProxy(); // Reset contexts so that they pick up the updated configuration. reset(); }