/** * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
/** * Helper function to select executors. * * @param blockLocations where the block is store currently * @param numReplicas how many replicas to replicate or evict * @param workerInfoList a list of current available job workers * @return the selection result */ private Map<WorkerInfo, SerializableVoid> selectExecutorsTestHelper( List<BlockLocation> blockLocations, int numReplicas, List<WorkerInfo> workerInfoList) throws Exception { BlockInfo blockInfo = new BlockInfo().setBlockId(TEST_BLOCK_ID); blockInfo.setLocations(blockLocations); when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(blockInfo); PowerMockito.mockStatic(AlluxioBlockStore.class); when(AlluxioBlockStore.create(mMockFileSystemContext)).thenReturn(mMockBlockStore); String path = "/test"; ReplicateConfig config = new ReplicateConfig(path, TEST_BLOCK_ID, numReplicas); ReplicateDefinition definition = new ReplicateDefinition(mMockFileSystemContext, mMockFileSystem); return definition.selectExecutors(config, workerInfoList, mMockJobMasterContext); }
/** * Converts a proto type to a wire type. * * @param blockPInfo the proto type to convert * @return the converted wire type */ public static BlockInfo fromProto(alluxio.grpc.BlockInfo blockPInfo) { BlockInfo blockInfo = new BlockInfo(); blockInfo.setBlockId(blockPInfo.getBlockId()); blockInfo.setLength(blockPInfo.getLength()); blockInfo.setLocations(map(GrpcUtils::fromProto, blockPInfo.getLocationsList())); return blockInfo; }
/** * Gets the in-Alluxio percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in Alluxio, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in alluxio percentage */ private int getInAlluxioPercentage(Inode inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = inode.asFile(); long length = inodeFile.getLength(); if (length == 0) { return 100; } long inAlluxioLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (!info.getLocations().isEmpty()) { inAlluxioLength += info.getLength(); } } return (int) (inAlluxioLength * 100 / length); }
/** * @param blockWorkers a list of block workers * @param blockInfo information about a block * @return the block worker hosts which are not storing the specified block */ private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers, FileBlockInfo blockInfo) { List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream() .map(location -> location.getWorkerAddress().getHost()) .collect(Collectors.toList()); return blockWorkers.stream() .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost())) .map(worker -> worker.getNetAddress().getHost()) .collect(Collectors.toList()); }
private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception { FileInfo testFileInfo = new FileInfo(); AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int i = 0; i < numOfBlocks; i++) { blockInfos.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList()))); } testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); Mockito.when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; } }
for (FileBlockInfo blockInfo : mFileSystem.getStatus(uri).getFileBlockInfos()) { List<String> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo); int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size(); if (workersWithoutBlock.size() < neededReplicas) { String missingJobWorkersMessage = ""; String address = workersWithoutBlock.get(i); WorkerInfo jobWorker = jobWorkersByAddress.get(address); assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId()));
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }
/** * Gets a stream to write data to a block. The stream can only be backed by Alluxio storage. * * @param blockId the block to write * @param blockSize the standard block size to write, or -1 if the block already exists (and this * stream is just storing the block in Alluxio again) * @param address the address of the worker to write the block to, fails if the worker cannot * serve the request * @param options the output stream options * @return an {@link BlockOutStream} which can be used to write data to the block in a streaming * fashion */ public BlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address, OutStreamOptions options) throws IOException { if (blockSize == -1) { try (CloseableResource<BlockMasterClient> blockMasterClientResource = mContext.acquireBlockMasterClientResource()) { blockSize = blockMasterClientResource.get().getBlockInfo(blockId).getLength(); } } // No specified location to write to. if (address == null) { throw new ResourceExhaustedException(ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER .getMessage(FormatUtils.getSizeFromBytes(blockSize))); } LOG.debug("Create block outstream for {} of block size {} at address {}, using options: {}", blockId, blockSize, address, options); return BlockOutStream.create(mContext, blockId, blockSize, address, options); }
ReadType readType = ReadType.fromProto(options.getOptions().getReadType()); long blockId = info.getBlockId(); long blockSize = info.getLength();
/** * @param blockId id of the block * @return the block info associated with the block id, note that this will be a cached copy * and will not fetch the latest info from the master */ public BlockInfo getBlockInfo(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); return mStatus.getFileBlockInfos().stream().map(FileBlockInfo::getBlockInfo) .filter(blockInfo -> blockInfo.getBlockId() == blockId).findFirst().get(); }
@Test public void getInStreamMissingBlock() throws Exception { URIStatus dummyStatus = new URIStatus( new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(NotFoundException.class); mException.expectMessage("unavailable in both Alluxio and UFS"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations() .stream().map(alluxio.wire.BlockLocation::getWorkerAddress).collect(toList()); if (locations.isEmpty()) { // No in-Alluxio location String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new); blockLocations.add( new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));
/** * Creates a file with the given name and a block on each specified worker. Workers may be * repeated to give them multiple blocks. * * @param testFile the name of the file to create * @param fileInfo file info to apply to the created file * @param workerInds the workers to put blocks on, specified by their indices * @return file info for the created file */ private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo, int... workerInds) throws Exception { AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int workerInd : workerInds) { WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress(); blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo() .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address))))); } FileInfo testFileInfo = fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; }
/** * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to * the block, such as the file offset, and additional UFS locations for the block. * * @param inodePath the file the block is a part of * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from * @return a new {@link FileBlockInfo} for the block */ private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws FileDoesNotExistException { InodeFile file = inodePath.getInodeFile(); FileBlockInfo fileBlockInfo = new FileBlockInfo(); fileBlockInfo.setBlockInfo(blockInfo); fileBlockInfo.setUfsLocations(new ArrayList<>()); // The sequence number part of the block id is the block index. long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId()); fileBlockInfo.setOffset(offset); if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) { // No alluxio locations, but there is a checkpoint in the under storage system. Add the // locations from the under storage system. long blockId = fileBlockInfo.getBlockInfo().getBlockId(); List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(), fileBlockInfo.getOffset()); if (locations != null) { fileBlockInfo.setUfsLocations(locations); } } return fileBlockInfo; }
@Before public void before() throws Exception { BlockWorkerClient workerClient = PowerMockito.mock(BlockWorkerClient.class); ClientCallStreamObserver requestObserver = PowerMockito.mock(ClientCallStreamObserver.class); when(requestObserver.isReady()).thenReturn(true); when(workerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build()); observer.onCompleted(); return requestObserver; } }); mMockContext = PowerMockito.mock(FileSystemContext.class); PowerMockito.when(mMockContext.acquireBlockWorkerClient(Matchers.any(WorkerNetAddress.class))) .thenReturn(workerClient); PowerMockito.when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); PowerMockito.when(mMockContext.getConf()).thenReturn(mConf); PowerMockito.doNothing().when(mMockContext) .releaseBlockWorkerClient(Matchers.any(WorkerNetAddress.class), Matchers.any(BlockWorkerClient.class)); mInfo = new BlockInfo().setBlockId(1); mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections .singletonList(1L))), mConf); }
/** * Gets the in-memory percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in memory percentage */ private int getInMemoryPercentage(Inode inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = inode.asFile(); long length = inodeFile.getLength(); if (length == 0) { return 100; } long inMemoryLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (isInTopStorageTier(info)) { inMemoryLength += info.getLength(); } } return (int) (inMemoryLength * 100 / length); }
for (FileBlockInfo fileBlockInfo : mFileSystemMasterView .getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) { blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
@Test public void getInStreamNoWorkers() throws Exception { URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(UnavailableException.class); mException.expectMessage("No Alluxio worker available"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }