How to use

BlockInfo

in

alluxio.wire

/**
 * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}.
 *
 * @param fileBlockInfo underlying {@link FileBlockInfo}
 */
public UIFileBlockInfo(FileBlockInfo fileBlockInfo) {
 Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo");
 mId = fileBlockInfo.getBlockInfo().getBlockId();
 mBlockLength = fileBlockInfo.getBlockInfo().getLength();
 mLastAccessTimeMs = -1;
 addLocations(fileBlockInfo);
 for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
  mTierAliases.add(location.getTierAlias());
 }
}

/**
 * Helper function to select executors.
 *
 * @param blockLocations where the block is store currently
 * @param numReplicas how many replicas to replicate or evict
 * @param workerInfoList a list of current available job workers
 * @return the selection result
 */
private Map<WorkerInfo, SerializableVoid> selectExecutorsTestHelper(
  List<BlockLocation> blockLocations, int numReplicas, List<WorkerInfo> workerInfoList)
  throws Exception {
 BlockInfo blockInfo = new BlockInfo().setBlockId(TEST_BLOCK_ID);
 blockInfo.setLocations(blockLocations);
 when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(blockInfo);
 PowerMockito.mockStatic(AlluxioBlockStore.class);
 when(AlluxioBlockStore.create(mMockFileSystemContext)).thenReturn(mMockBlockStore);
 String path = "/test";
 ReplicateConfig config = new ReplicateConfig(path, TEST_BLOCK_ID, numReplicas);
 ReplicateDefinition definition = new ReplicateDefinition(mMockFileSystemContext,
   mMockFileSystem);
 return definition.selectExecutors(config, workerInfoList, mMockJobMasterContext);
}

/**
 * Converts a proto type to a wire type.
 *
 * @param blockPInfo the proto type to convert
 * @return the converted wire type
 */
public static BlockInfo fromProto(alluxio.grpc.BlockInfo blockPInfo) {
 BlockInfo blockInfo = new BlockInfo();
 blockInfo.setBlockId(blockPInfo.getBlockId());
 blockInfo.setLength(blockPInfo.getLength());
 blockInfo.setLocations(map(GrpcUtils::fromProto, blockPInfo.getLocationsList()));
 return blockInfo;
}

/**
 * Gets the in-Alluxio percentage of an Inode. For a file that has all blocks in Alluxio, it
 * returns 100; for a file that has no block in Alluxio, it returns 0. Returns 0 for a directory.
 *
 * @param inode the inode
 * @return the in alluxio percentage
 */
private int getInAlluxioPercentage(Inode inode) throws UnavailableException {
 if (!inode.isFile()) {
  return 0;
 }
 InodeFile inodeFile = inode.asFile();
 long length = inodeFile.getLength();
 if (length == 0) {
  return 100;
 }
 long inAlluxioLength = 0;
 for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) {
  if (!info.getLocations().isEmpty()) {
   inAlluxioLength += info.getLength();
  }
 }
 return (int) (inAlluxioLength * 100 / length);
}

/**
 * @param blockWorkers a list of block workers
 * @param blockInfo information about a block
 * @return the block worker hosts which are not storing the specified block
 */
private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers,
  FileBlockInfo blockInfo) {
 List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream()
   .map(location -> location.getWorkerAddress().getHost())
   .collect(Collectors.toList());
 return blockWorkers.stream()
   .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost()))
   .map(worker -> worker.getNetAddress().getHost())
   .collect(Collectors.toList());
}

 private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception {
  FileInfo testFileInfo = new FileInfo();
  AlluxioURI uri = new AlluxioURI(testFile);
  List<FileBlockInfo> blockInfos = Lists.newArrayList();
  for (int i = 0; i < numOfBlocks; i++) {
   blockInfos.add(new FileBlockInfo()
     .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList())));
  }
  testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos);
  Mockito.when(mMockFileSystem.listStatus(uri))
    .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo)));
  Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
  return testFileInfo;
 }
}

for (FileBlockInfo blockInfo : mFileSystem.getStatus(uri).getFileBlockInfos()) {
 List<String> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo);
 int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size();
 if (workersWithoutBlock.size() < neededReplicas) {
  String missingJobWorkersMessage = "";
  String address = workersWithoutBlock.get(i);
  WorkerInfo jobWorker = jobWorkersByAddress.get(address);
  assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId()));

 @Test
 public void selectExecutorsMissingLocationTest() throws Exception {
  AlluxioURI uri = new AlluxioURI("/test");
  PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");

  long blockId = 1;
  BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
  FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
  FileInfo testFileInfo = new FileInfo();
  testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
  Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));

  try {
   new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config,
     Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext);
  } catch (Exception e) {
   Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage());
  }
 }
}

/**
 * Gets a stream to write data to a block. The stream can only be backed by Alluxio storage.
 *
 * @param blockId the block to write
 * @param blockSize the standard block size to write, or -1 if the block already exists (and this
 *        stream is just storing the block in Alluxio again)
 * @param address the address of the worker to write the block to, fails if the worker cannot
 *        serve the request
 * @param options the output stream options
 * @return an {@link BlockOutStream} which can be used to write data to the block in a streaming
 *         fashion
 */
public BlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address,
  OutStreamOptions options) throws IOException {
 if (blockSize == -1) {
  try (CloseableResource<BlockMasterClient> blockMasterClientResource =
    mContext.acquireBlockMasterClientResource()) {
   blockSize = blockMasterClientResource.get().getBlockInfo(blockId).getLength();
  }
 }
 // No specified location to write to.
 if (address == null) {
  throw new ResourceExhaustedException(ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER
    .getMessage(FormatUtils.getSizeFromBytes(blockSize)));
 }
 LOG.debug("Create block outstream for {} of block size {} at address {}, using options: {}",
   blockId, blockSize, address, options);
 return BlockOutStream.create(mContext, blockId, blockSize, address, options);
}

ReadType readType = ReadType.fromProto(options.getOptions().getReadType());
long blockId = info.getBlockId();
long blockSize = info.getLength();

/**
 * @param blockId id of the block
 * @return the block info associated with the block id, note that this will be a cached copy
 * and will not fetch the latest info from the master
 */
public BlockInfo getBlockInfo(long blockId) {
 Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId");
 return mStatus.getFileBlockInfos().stream().map(FileBlockInfo::getBlockInfo)
   .filter(blockInfo -> blockInfo.getBlockId() == blockId).findFirst().get();
}

@Test
public void getInStreamMissingBlock() throws Exception {
 URIStatus dummyStatus = new URIStatus(
   new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L)));
 InStreamOptions options =
   new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf);
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
 when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList());
 mException.expect(NotFoundException.class);
 mException.expectMessage("unavailable in both Alluxio and UFS");
 mBlockStore.getInStream(BLOCK_ID, options).getAddress();
}

private void addLocations(FileBlockInfo fileBlockInfo) {
 Set<String> locations = new HashSet<>();
 // add alluxio locations
 for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
  locations.add(location.getWorkerAddress().getHost());
 }
 // add underFS locations
 for (String location : fileBlockInfo.getUfsLocations()) {
  locations.add(HostAndPort.fromString(location).getHost());
 }
 mLocations.addAll(locations);
}

for (FileBlockInfo fileBlockInfo : blocks) {
 long offset = fileBlockInfo.getOffset();
 long end = offset + fileBlockInfo.getBlockInfo().getLength();
  List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations()
    .stream().map(alluxio.wire.BlockLocation::getWorkerAddress).collect(toList());
  if (locations.isEmpty()) { // No in-Alluxio location
  String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new);
  blockLocations.add(
    new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));

/**
 * Creates a file with the given name and a block on each specified worker. Workers may be
 * repeated to give them multiple blocks.
 *
 * @param testFile the name of the file to create
 * @param fileInfo file info to apply to the created file
 * @param workerInds the workers to put blocks on, specified by their indices
 * @return file info for the created file
 */
private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo,
  int... workerInds) throws Exception {
 AlluxioURI uri = new AlluxioURI(testFile);
 List<FileBlockInfo> blockInfos = Lists.newArrayList();
 for (int workerInd : workerInds) {
  WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress();
  blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo()
    .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address)))));
 }
 FileInfo testFileInfo =
   fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos);
 when(mMockFileSystem.listStatus(uri))
   .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo)));
 when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
 return testFileInfo;
}

/**
 * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to
 * the block, such as the file offset, and additional UFS locations for the block.
 *
 * @param inodePath the file the block is a part of
 * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from
 * @return a new {@link FileBlockInfo} for the block
 */
private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo)
  throws FileDoesNotExistException {
 InodeFile file = inodePath.getInodeFile();
 FileBlockInfo fileBlockInfo = new FileBlockInfo();
 fileBlockInfo.setBlockInfo(blockInfo);
 fileBlockInfo.setUfsLocations(new ArrayList<>());
 // The sequence number part of the block id is the block index.
 long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId());
 fileBlockInfo.setOffset(offset);
 if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) {
  // No alluxio locations, but there is a checkpoint in the under storage system. Add the
  // locations from the under storage system.
  long blockId = fileBlockInfo.getBlockInfo().getBlockId();
  List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(),
    fileBlockInfo.getOffset());
  if (locations != null) {
   fileBlockInfo.setUfsLocations(locations);
  }
 }
 return fileBlockInfo;
}

@Before
public void before() throws Exception {
 BlockWorkerClient workerClient = PowerMockito.mock(BlockWorkerClient.class);
 ClientCallStreamObserver requestObserver = PowerMockito.mock(ClientCallStreamObserver.class);
 when(requestObserver.isReady()).thenReturn(true);
 when(workerClient.openLocalBlock(any(StreamObserver.class)))
   .thenAnswer(new Answer() {
    public Object answer(InvocationOnMock invocation) {
     StreamObserver<OpenLocalBlockResponse> observer =
       invocation.getArgumentAt(0, StreamObserver.class);
     observer.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build());
     observer.onCompleted();
     return requestObserver;
    }
   });
 mMockContext = PowerMockito.mock(FileSystemContext.class);
 PowerMockito.when(mMockContext.acquireBlockWorkerClient(Matchers.any(WorkerNetAddress.class)))
   .thenReturn(workerClient);
 PowerMockito.when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf));
 PowerMockito.when(mMockContext.getConf()).thenReturn(mConf);
 PowerMockito.doNothing().when(mMockContext)
   .releaseBlockWorkerClient(Matchers.any(WorkerNetAddress.class),
     Matchers.any(BlockWorkerClient.class));
 mInfo = new BlockInfo().setBlockId(1);
 mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections
   .singletonList(1L))), mConf);
}

/**
 * Gets the in-memory percentage of an Inode. For a file that has all blocks in Alluxio, it
 * returns 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory.
 *
 * @param inode the inode
 * @return the in memory percentage
 */
private int getInMemoryPercentage(Inode inode) throws UnavailableException {
 if (!inode.isFile()) {
  return 0;
 }
 InodeFile inodeFile = inode.asFile();
 long length = inodeFile.getLength();
 if (length == 0) {
  return 100;
 }
 long inMemoryLength = 0;
 for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) {
  if (isInTopStorageTier(info)) {
   inMemoryLength += info.getLength();
  }
 }
 return (int) (inMemoryLength * 100 / length);
}

for (FileBlockInfo fileBlockInfo : mFileSystemMasterView
  .getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) {
 blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());

@Test
public void getInStreamNoWorkers() throws Exception {
 URIStatus dummyStatus =
   new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)));
 InStreamOptions options =
   new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(),
     sConf);
 when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
 when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList());
 mException.expect(UnavailableException.class);
 mException.expectMessage("No Alluxio worker available");
 mBlockStore.getInStream(BLOCK_ID, options).getAddress();
}