How to use

alluxio.wire.FileInfo

constructor

/**
 * Creates a mock {@link FileInStream} which will supply the given bytes.
 *
 * @param context file system context
 * @param bytes the bytes to supply
 */
public MockFileInStream(FileSystemContext context, byte[] bytes, AlluxioConfiguration conf) {
 super(new URIStatus(new FileInfo()), new InStreamOptions(new URIStatus(new FileInfo()),
   conf), context);
 mStream = new ByteArrayInputStream(bytes);
}

private FileInfo createFileWithBlocksOnWorkers(String testFile, int... workerInds)
  throws Exception {
 return createFileWithBlocksOnWorkers(testFile, new FileInfo(), workerInds);
}

/**
 * Tests that the worker will not delete the source directory if the directory still contains
 * files.
 */
@Test
public void dontDeleteNonEmptySourceTest() throws Exception {
 when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR)))
   .thenReturn(Lists.newArrayList(new URIStatus(new FileInfo())));
 runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH);
 verify(mMockFileSystem, times(0)).delete(eq(new AlluxioURI(TEST_DIR)),
   any(DeletePOptions.class));
}

/**
 * Tests that the worker will delete the source directory if the directory contains only
 * directories.
 */
@Test
public void deleteDirsOnlySourceDir() throws Exception {
 String inner = TEST_DIR + "/innerDir";
 when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))).thenReturn(
   Lists.newArrayList(new URIStatus(new FileInfo().setPath(inner).setFolder(true))));
 when(mMockFileSystem.listStatus(new AlluxioURI(inner)))
   .thenReturn(Lists.<URIStatus>newArrayList());
 runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH);
 verify(mMockFileSystem).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class));
}

/**
 * Tests for the {@link BaseFileSystem#listStatus(AlluxioURI, ListStatusPOptions)} method.
 */
@Test
public void listStatus() throws Exception {
 AlluxioURI file = new AlluxioURI("/file");
 List<URIStatus> infos = new ArrayList<>();
 infos.add(new URIStatus(new FileInfo()));
 ListStatusPOptions listStatusOptions = ListStatusPOptions.getDefaultInstance();
 when(mFileSystemMasterClient.listStatus(file, listStatusOptions)).thenReturn(infos);
 assertSame(infos, mFileSystem.listStatus(file, listStatusOptions));
 verify(mFileSystemMasterClient).listStatus(file, listStatusOptions);
 verifyFilesystemContextAcquiredAndReleased();
}

@Test
public void getInStreamMissingBlock() throws Exception {
 URIStatus dummyStatus = new URIStatus(
   new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L)));
 InStreamOptions options =
   new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf);
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
 when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList());
 mException.expect(NotFoundException.class);
 mException.expectMessage("unavailable in both Alluxio and UFS");
 mBlockStore.getInStream(BLOCK_ID, options).getAddress();
}

/**
 * Tests for the {@link BaseFileSystem#openFile(AlluxioURI, OpenFilePOptions)} method to
 * complete successfully.
 */
@Test
public void openFile() throws Exception {
 AlluxioURI file = new AlluxioURI("/file");
 URIStatus status = new URIStatus(new FileInfo());
 GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance();
 when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status);
 mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance());
 verify(mFileSystemMasterClient).getStatus(file, getStatusOptions);
 verifyFilesystemContextAcquiredAndReleased();
}

/**
 * Tests for the {@link BaseFileSystem#getStatus(AlluxioURI, GetStatusPOptions)} method.
 */
@Test
public void getStatus() throws Exception {
 AlluxioURI file = new AlluxioURI("/file");
 URIStatus status = new URIStatus(new FileInfo());
 GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance();
 when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status);
 assertSame(status, mFileSystem.getStatus(file, getStatusOptions));
 verify(mFileSystemMasterClient).getStatus(file, getStatusOptions);
 verifyFilesystemContextAcquiredAndReleased();
}

@Test
public void getInStreamNoWorkers() throws Exception {
 URIStatus dummyStatus =
   new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)));
 InStreamOptions options =
   new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(),
     sConf);
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
 when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList());
 mException.expect(UnavailableException.class);
 mException.expectMessage("No Alluxio worker available");
 mBlockStore.getInStream(BLOCK_ID, options).getAddress();
}

 private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception {
  FileInfo testFileInfo = new FileInfo();
  AlluxioURI uri = new AlluxioURI(testFile);
  List<FileBlockInfo> blockInfos = Lists.newArrayList();
  for (int i = 0; i < numOfBlocks; i++) {
   blockInfos.add(new FileBlockInfo()
     .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList())));
  }
  testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos);
  Mockito.when(mMockFileSystem.listStatus(uri))
    .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo)));
  Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
  return testFileInfo;
 }
}

/**
 * Creates a directory with the given name.
 *
 * @return file info for the created directory
 */
private FileInfo createDirectory(String name) throws Exception {
 // Call all directories mount points to force cross-mount functionality.
 FileInfo info = new FileInfo().setFolder(true).setPath(name).setMountPoint(true);
 when(mMockFileSystem.getStatus(new AlluxioURI(name))).thenReturn(new URIStatus(info));
 return info;
}

/**
 * Tests the creation of a file via the
 * {@link BaseFileSystem#createFile(AlluxioURI, CreateFilePOptions)} method.
 */
@Test
public void createFile() throws Exception {
 doNothing().when(mFileSystemMasterClient)
   .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class));
 URIStatus status = new URIStatus(new FileInfo());
 AlluxioURI file = new AlluxioURI("/file");
 GetStatusPOptions getStatusOptions =
   GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).build();
 when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status);
 FileOutStream out = mFileSystem.createFile(file, CreateFilePOptions.getDefaultInstance());
 verify(mFileSystemMasterClient).createFile(file, CreateFilePOptions.getDefaultInstance());
 assertEquals(out.mUri, file);
 verifyFilesystemContextAcquiredAndReleased();
}

@Test
public void getInStreamRemote() throws Exception {
 WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1");
 WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2");
 BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays
   .asList(new BlockLocation().setWorkerAddress(remote1),
     new BlockLocation().setWorkerAddress(remote2)));
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
 // We should sometimes get remote1 and sometimes get remote2.
 Set<WorkerNetAddress> results = new HashSet<>();
 for (int i = 0; i < 40; i++) {
  results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(
    new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf))
    .getAddress());
 }
 assertEquals(Sets.newHashSet(remote1, remote2), results);
}

@Test
public void scheduleAsyncPersist() throws Exception {
 DefaultAsyncPersistHandler handler =
   new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
 AlluxioURI path = new AlluxioURI("/test");
 long blockId = 0;
 long workerId = 1;
 long fileId = 2;
 List<FileBlockInfo> blockInfoList = new ArrayList<>();
 BlockLocation location = new BlockLocation().setWorkerId(workerId);
 blockInfoList.add(new FileBlockInfo().setBlockInfo(
   new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
 when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
 when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
 when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
 when(mFileSystemMaster.getFileInfo(fileId))
   .thenReturn(new FileInfo().setLength(1).setCompleted(true));
 handler.scheduleAsyncPersistence(path);
 List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
 assertEquals(1, persistFiles.size());
 assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}

@Test
public void getInStreamLocal() throws Exception {
 WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
 WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
 // Mock away gRPC usage.
 OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build();
 when(mWorkerClient.openLocalBlock(any(StreamObserver.class)))
     .thenAnswer(new Answer() {
      public Object answer(InvocationOnMock invocation) {
       StreamObserver<OpenLocalBlockResponse> observer =
           invocation.getArgumentAt(0, StreamObserver.class);
       observer.onNext(response);
       observer.onCompleted();
       return mStreamObserver;
      }
     });
 BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays
   .asList(new BlockLocation().setWorkerAddress(remote),
     new BlockLocation().setWorkerAddress(local)));
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
 assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(
   new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf))
             .getAddress());
}

 @Test
 public void selectExecutorsMissingLocationTest() throws Exception {
  AlluxioURI uri = new AlluxioURI("/test");
  PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");

  long blockId = 1;
  BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
  FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
  FileInfo testFileInfo = new FileInfo();
  testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
  Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));

  try {
   new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config,
     Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext);
  } catch (Exception e) {
   Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage());
  }
 }
}

/**
 * Tests the persistence of file with block on multiple workers.
 */
@Test
public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception {
 DefaultAsyncPersistHandler handler =
   new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
 AlluxioURI path = new AlluxioURI("/test");
 List<FileBlockInfo> blockInfoList = new ArrayList<>();
 BlockLocation location1 = new BlockLocation().setWorkerId(1);
 blockInfoList.add(new FileBlockInfo()
   .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1))));
 BlockLocation location2 = new BlockLocation().setWorkerId(2);
 blockInfoList.add(new FileBlockInfo()
   .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2))));
 long fileId = 2;
 when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
 when(mFileSystemMaster.getFileInfo(fileId))
   .thenReturn(new FileInfo().setLength(1).setCompleted(true));
 when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
 // no persist scheduled on any worker
 assertEquals(0, handler.pollFilesToPersist(1).size());
 assertEquals(0, handler.pollFilesToPersist(2).size());
}

@Test
public void getStatus() throws Exception {
 FileInfo fileInfo = new FileInfo()
   .setLastModificationTimeMs(111L)
   .setFolder(false)
   .setOwner("user1")
   .setGroup("group1")
   .setMode(00755);
 Path path = new Path("/dir");
 alluxio.client.file.FileSystem alluxioFs =
   mock(alluxio.client.file.FileSystem.class);
 when(alluxioFs.getStatus(new AlluxioURI(HadoopUtils.getPathWithoutScheme(path))))
   .thenReturn(new URIStatus(fileInfo));
 FileSystem alluxioHadoopFs = new FileSystem(alluxioFs);
 FileStatus fileStatus = alluxioHadoopFs.getFileStatus(path);
 assertFileInfoEqualsFileStatus(fileInfo, fileStatus);
}

@Test
public void selectExecutorsTest() throws Exception {
 AlluxioURI uri = new AlluxioURI("/test");
 PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
 WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
 WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
 long blockId = 1;
 BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
 FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
 BlockLocation location = new BlockLocation();
 location.setWorkerAddress(workerNetAddress);
 blockInfo.setLocations(Lists.newArrayList(location));
 FileInfo testFileInfo = new FileInfo();
 testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
 Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
 Map<WorkerInfo, SerializableVoid> result =
   new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config,
     Lists.newArrayList(workerInfo), mMockJobMasterContext);
 Assert.assertEquals(1, result.size());
 Assert.assertEquals(workerInfo, result.keySet().iterator().next());
}

@Test
public void getInStreamUfs() throws Exception {
 WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1");
 WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2");
 BlockInfo info = new BlockInfo().setBlockId(0);
 URIStatus dummyStatus =
   new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))
     .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
 OpenFilePOptions readOptions = OpenFilePOptions.newBuilder()
   .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build();
 InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
 ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy())
   .setHosts(Arrays.asList(worker1, worker2));
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
 when(mMasterClient.getWorkerInfoList()).thenReturn(
   Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2)));
 // Location policy chooses worker1 first.
 assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
 // Location policy chooses worker2 second.
 assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
}