/** * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
/** * @param blockWorkers a list of block workers * @param blockInfo information about a block * @return the block worker hosts which are not storing the specified block */ private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers, FileBlockInfo blockInfo) { List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream() .map(location -> location.getWorkerAddress().getHost()) .collect(Collectors.toList()); return blockWorkers.stream() .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost())) .map(worker -> worker.getNetAddress().getHost()) .collect(Collectors.toList()); }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations() .stream().map(alluxio.wire.BlockLocation::getWorkerAddress).collect(toList()); if (locations.isEmpty()) { // No in-Alluxio location String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new); blockLocations.add( new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));
for (FileBlockInfo fileBlockInfo : mFileSystemMasterView .getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) { blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
BlockWorkerInfo mostBlocksWorker = null; for (FileBlockInfo fileBlockInfo : fileBlockInfos) { for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { BlockWorkerInfo worker = addressIndexedWorkers.getFirstByField(WORKER_ADDRESS_INDEX, location.getWorkerAddress());
for (BlockLocation blockLocation : fileBlockInfo.getBlockInfo().getLocations()) { if (workerBlockCounts.containsKey(blockLocation.getWorkerId())) { workerBlockCounts.put(blockLocation.getWorkerId(),
for (FileBlockInfo blockInfo : mFileSystem.getStatus(uri).getFileBlockInfos()) { List<String> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo); int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size(); if (workersWithoutBlock.size() < neededReplicas) { String missingJobWorkersMessage = ""; String address = workersWithoutBlock.get(i); WorkerInfo jobWorker = jobWorkersByAddress.get(address); assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId()));
/** * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to * the block, such as the file offset, and additional UFS locations for the block. * * @param inodePath the file the block is a part of * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from * @return a new {@link FileBlockInfo} for the block */ private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws FileDoesNotExistException { InodeFile file = inodePath.getInodeFile(); FileBlockInfo fileBlockInfo = new FileBlockInfo(); fileBlockInfo.setBlockInfo(blockInfo); fileBlockInfo.setUfsLocations(new ArrayList<>()); // The sequence number part of the block id is the block index. long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId()); fileBlockInfo.setOffset(offset); if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) { // No alluxio locations, but there is a checkpoint in the under storage system. Add the // locations from the under storage system. long blockId = fileBlockInfo.getBlockInfo().getBlockId(); List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(), fileBlockInfo.getOffset()); if (locations != null) { fileBlockInfo.setUfsLocations(locations); } } return fileBlockInfo; }
public void checkEquality(FileBlockInfo a, FileBlockInfo b) { Assert.assertEquals(a.getBlockInfo(), b.getBlockInfo()); Assert.assertEquals(a.getOffset(), b.getOffset()); Assert.assertEquals(a.getUfsLocations(), b.getUfsLocations()); Assert.assertEquals(a, b); }
/** * Converts wire type to proto type. * * @param fileBlockInfo the wire representation to convert * @return converted proto representation */ public static alluxio.grpc.FileBlockInfo toProto(FileBlockInfo fileBlockInfo) { List<alluxio.grpc.WorkerNetAddress> ufsLocations = new ArrayList<>(); for (String ufsLocation : fileBlockInfo.getUfsLocations()) { HostAndPort address = HostAndPort.fromString(ufsLocation); ufsLocations.add(alluxio.grpc.WorkerNetAddress.newBuilder().setHost(address.getHost()) .setDataPort(address.getPortOrDefault(-1)).build()); } return alluxio.grpc.FileBlockInfo.newBuilder() .setBlockInfo(toProto(fileBlockInfo.getBlockInfo())).setOffset(fileBlockInfo.getOffset()) .addAllUfsLocations(ufsLocations).addAllUfsStringLocations(fileBlockInfo.getUfsLocations()) .build(); }
boolean overReplicated = mStatus.getReplicationMax() > 0 && mStatus.getFileBlockInfos().get((int) (getPos() / mBlockSize)) .getBlockInfo().getLocations().size() >= mStatus.getReplicationMax(); cache = cache && !overReplicated;
List<String> locations = new ArrayList<>(); for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) { WorkerNetAddress address = location.getWorkerAddress(); locations.add(address.getHost() + ":" + address.getDataPort());
/** * Creates a new instance of {@link UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHostText()); } mLocations.addAll(locations); }
@Override public void reportLostFile(long fileId) throws FileDoesNotExistException, UnavailableException { try ( LockedInodePath inodePath = mInodeTree.lockFullInodePath(fileId, InodeTree.LockMode.READ)) { Inode<?> inode = inodePath.getInode(); if (inode.isDirectory()) { LOG.warn("Reported file is a directory {}", inode); return; } List<Long> blockIds = new ArrayList<>(); try { for (FileBlockInfo fileBlockInfo : getFileBlockInfoListInternal(inodePath)) { blockIds.add(fileBlockInfo.getBlockInfo().getBlockId()); } } catch (InvalidPathException e) { LOG.info("Failed to get file info {}", fileId, e); } mBlockMaster.reportLostBlocks(blockIds); LOG.info("Reported file loss of blocks {}. Alluxio will recompute it: {}", blockIds, fileId); } }
for (FileBlockInfo fileBlockInfo : mFileSystemMasterView .getFileBlockInfoList(mFileSystemMasterView.getPath(fileId))) { blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations() .stream().map(alluxio.wire.BlockLocation::getWorkerAddress).collect(toList()); if (locations.isEmpty()) { String[] hosts = addresses.stream().map(HostAndPort::getHostText).toArray(String[]::new); blockLocations.add( new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));
FileBlockInfo info = blocks.get(k); long offset = info.getOffset(); long end = offset + info.getBlockInfo().getLength(); List<WorkerNetAddress> addrs = Lists.newArrayList(); for (alluxio.wire.BlockLocation location : info.getBlockInfo().getLocations()) { addrs.add(location.getWorkerAddress()); CommonUtils.toStringArray(hosts), offset, info.getBlockInfo().getLength()));
/** * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to * the block, such as the file offset, and additional UFS locations for the block. * * @param inodePath the file the block is a part of * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from * @return a new {@link FileBlockInfo} for the block * @throws InvalidPathException if the mount table is not able to resolve the file */ private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws InvalidPathException, FileDoesNotExistException { InodeFile file = inodePath.getInodeFile(); FileBlockInfo fileBlockInfo = new FileBlockInfo(); fileBlockInfo.setBlockInfo(blockInfo); fileBlockInfo.setUfsLocations(new ArrayList<>()); // The sequence number part of the block id is the block index. long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId()); fileBlockInfo.setOffset(offset); if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) { // No alluxio locations, but there is a checkpoint in the under storage system. Add the // locations from the under storage system. long blockId = fileBlockInfo.getBlockInfo().getBlockId(); List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(), fileBlockInfo.getOffset()); if (locations != null) { fileBlockInfo.setUfsLocations(locations); } } return fileBlockInfo; }