/** * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to * the block, such as the file offset, and additional UFS locations for the block. * * @param inodePath the file the block is a part of * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from * @return a new {@link FileBlockInfo} for the block */ private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws FileDoesNotExistException { InodeFile file = inodePath.getInodeFile(); FileBlockInfo fileBlockInfo = new FileBlockInfo(); fileBlockInfo.setBlockInfo(blockInfo); fileBlockInfo.setUfsLocations(new ArrayList<>()); // The sequence number part of the block id is the block index. long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId()); fileBlockInfo.setOffset(offset); if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) { // No alluxio locations, but there is a checkpoint in the under storage system. Add the // locations from the under storage system. long blockId = fileBlockInfo.getBlockInfo().getBlockId(); List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(), fileBlockInfo.getOffset()); if (locations != null) { fileBlockInfo.setUfsLocations(locations); } } return fileBlockInfo; }
public void checkEquality(FileBlockInfo a, FileBlockInfo b) { Assert.assertEquals(a.getBlockInfo(), b.getBlockInfo()); Assert.assertEquals(a.getOffset(), b.getOffset()); Assert.assertEquals(a.getUfsLocations(), b.getUfsLocations()); Assert.assertEquals(a, b); }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception { FileInfo testFileInfo = new FileInfo(); AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int i = 0; i < numOfBlocks; i++) { blockInfos.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList()))); } testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); Mockito.when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; } }
/** * Converts a proto type to a wire type. * * @param fileBlockPInfo the proto type to convert * @return the converted wire type */ public static FileBlockInfo fromProto(alluxio.grpc.FileBlockInfo fileBlockPInfo) { return new FileBlockInfo().setBlockInfo(fromProto(fileBlockPInfo.getBlockInfo())) .setOffset(fileBlockPInfo.getOffset()).setUfsLocations( fileBlockPInfo.getUfsLocationsCount() > 0 ? fileBlockPInfo.getUfsStringLocationsList() : map(addr -> HostAndPort.fromParts(addr.getHost(), addr.getDataPort()).toString(), fileBlockPInfo.getUfsLocationsList())); }
/** * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
void verifyBlockLocations(List<WorkerNetAddress> blockWorkers, List<String> ufsLocations, List<WorkerNetAddress> allWorkers, List<WorkerNetAddress> expectedWorkers) throws Exception { FileBlockInfo blockInfo = new FileBlockInfo().setBlockInfo( new BlockInfo().setLocations(blockWorkers.stream().map( addr -> new alluxio.wire.BlockLocation().setWorkerAddress(addr)).collect( toList()))).setUfsLocations(ufsLocations); FileInfo fileInfo = new FileInfo() .setLastModificationTimeMs(111L)
/** * Creates a file with the given name and a block on each specified worker. Workers may be * repeated to give them multiple blocks. * * @param testFile the name of the file to create * @param fileInfo file info to apply to the created file * @param workerInds the workers to put blocks on, specified by their indices * @return file info for the created file */ private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo, int... workerInds) throws Exception { AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int workerInd : workerInds) { WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress(); blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo() .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address))))); } FileInfo testFileInfo = fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; }
public static FileBlockInfo createRandom() { FileBlockInfo result = new FileBlockInfo(); Random random = new Random(); BlockInfo blockInfo = BlockInfoTest.createRandom(); long offset = random.nextLong(); List<String> ufsLocations = new ArrayList<>(); long numUfsLocations = random.nextInt(10); for (int i = 0; i < numUfsLocations; i++) { ufsLocations.add(HostAndPort.fromParts(CommonUtils.randomAlphaNumString(random.nextInt(10)), random.nextInt(Constants.MAX_PORT)).toString()); } result.setBlockInfo(blockInfo); result.setOffset(offset); result.setUfsLocations(ufsLocations); return result; } }
List<String> locations = new ArrayList<>(); for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) { WorkerNetAddress address = location.getWorkerAddress(); locations.add(address.getHost() + ":" + address.getDataPort()); locations.addAll(blockInfo.getUfsLocations()); toAdd.setFileLocations(locations);
/** * @param blockWorkers a list of block workers * @param blockInfo information about a block * @return the block worker hosts which are not storing the specified block */ private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers, FileBlockInfo blockInfo) { List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream() .map(location -> location.getWorkerAddress().getHost()) .collect(Collectors.toList()); return blockWorkers.stream() .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost())) .map(worker -> worker.getNetAddress().getHost()) .collect(Collectors.toList()); }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
/** * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to * the block, such as the file offset, and additional UFS locations for the block. * * @param inodePath the file the block is a part of * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from * @return a new {@link FileBlockInfo} for the block * @throws InvalidPathException if the mount table is not able to resolve the file */ private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws InvalidPathException, FileDoesNotExistException { InodeFile file = inodePath.getInodeFile(); FileBlockInfo fileBlockInfo = new FileBlockInfo(); fileBlockInfo.setBlockInfo(blockInfo); fileBlockInfo.setUfsLocations(new ArrayList<>()); // The sequence number part of the block id is the block index. long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId()); fileBlockInfo.setOffset(offset); if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) { // No alluxio locations, but there is a checkpoint in the under storage system. Add the // locations from the under storage system. long blockId = fileBlockInfo.getBlockInfo().getBlockId(); List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(), fileBlockInfo.getOffset()); if (locations != null) { fileBlockInfo.setUfsLocations(locations); } } return fileBlockInfo; }
List<BlockLocation> blockLocations = new ArrayList<>(); for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations() .stream().map(alluxio.wire.BlockLocation::getWorkerAddress).collect(toList()); if (locations.isEmpty()) { // No in-Alluxio location if (!fileBlockInfo.getUfsLocations().isEmpty()) { locations = fileBlockInfo.getUfsLocations().stream().map( location -> finalWorkerHosts.get(HostAndPort.fromString(location).getHost())) .filter(Objects::nonNull).collect(toList()); String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new); blockLocations.add( new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));
/** * Creates a new instance of {@link FileBlockInfo} from a thrift representation. * * @param info the thrift representation of a file block information * @return the instance */ public static FileBlockInfo fromThrift(alluxio.thrift.FileBlockInfo info) { return new FileBlockInfo() .setBlockInfo(BlockInfo.fromThrift(info.getBlockInfo())) .setOffset(info.getOffset()) .setUfsLocations(info.getUfsStringLocationsSize() > 0 ? info.getUfsStringLocations() : map(addr -> HostAndPort.fromParts(addr.getHost(), addr.getDataPort()).toString(), info.getUfsLocations())); }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHostText()); } mLocations.addAll(locations); }
for (FileBlockInfo fileBlockInfo : mFileSystemMasterView .getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) { blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
/** * Tests the persistence of file with block on multiple workers. */ @Test public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location1 = new BlockLocation().setWorkerId(1); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1)))); BlockLocation location2 = new BlockLocation().setWorkerId(2); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2)))); long fileId = 2; when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); // no persist scheduled on any worker assertEquals(0, handler.pollFilesToPersist(1).size()); assertEquals(0, handler.pollFilesToPersist(2).size()); }
/** * Converts wire type to proto type. * * @param fileBlockInfo the wire representation to convert * @return converted proto representation */ public static alluxio.grpc.FileBlockInfo toProto(FileBlockInfo fileBlockInfo) { List<alluxio.grpc.WorkerNetAddress> ufsLocations = new ArrayList<>(); for (String ufsLocation : fileBlockInfo.getUfsLocations()) { HostAndPort address = HostAndPort.fromString(ufsLocation); ufsLocations.add(alluxio.grpc.WorkerNetAddress.newBuilder().setHost(address.getHost()) .setDataPort(address.getPortOrDefault(-1)).build()); } return alluxio.grpc.FileBlockInfo.newBuilder() .setBlockInfo(toProto(fileBlockInfo.getBlockInfo())).setOffset(fileBlockInfo.getOffset()) .addAllUfsLocations(ufsLocations).addAllUfsStringLocations(fileBlockInfo.getUfsLocations()) .build(); }
List<String> locations = new ArrayList<>(); for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) { WorkerNetAddress address = location.getWorkerAddress(); locations.add(address.getHost() + ":" + address.getDataPort()); locations.addAll(blockInfo.getUfsLocations()); toAdd.setFileLocations(locations);