for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new); blockLocations.add( new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));
/** * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
/** * Converts wire type to proto type. * * @param blockInfo the wire type to convert * @return the converted proto type */ public static alluxio.grpc.BlockInfo toProto(BlockInfo blockInfo) { List<alluxio.grpc.BlockLocation> locations = new ArrayList<>(); for (BlockLocation location : blockInfo.getLocations()) { locations.add(toProto(location)); } return alluxio.grpc.BlockInfo.newBuilder().setBlockId(blockInfo.getBlockId()) .setLength(blockInfo.getLength()).addAllLocations(locations).build(); }
.filter(workerInfo -> workers.contains(workerInfo.getNetAddress())).collect(toList()); GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults().setBlockId(info.getBlockId()) .setBlockSize(info.getLength()).setBlockWorkerInfos(blockWorkerInfo); dataSource = policy.getWorker(getWorkerOptions);
/** * Gets the in-Alluxio percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in Alluxio, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in alluxio percentage */ private int getInAlluxioPercentage(Inode inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = inode.asFile(); long length = inodeFile.getLength(); if (length == 0) { return 100; } long inAlluxioLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (!info.getLocations().isEmpty()) { inAlluxioLength += info.getLength(); } } return (int) (inAlluxioLength * 100 / length); }
/** * Gets a stream to write data to a block. The stream can only be backed by Alluxio storage. * * @param blockId the block to write * @param blockSize the standard block size to write, or -1 if the block already exists (and this * stream is just storing the block in Alluxio again) * @param address the address of the worker to write the block to, fails if the worker cannot * serve the request * @param options the output stream options * @return an {@link BlockOutStream} which can be used to write data to the block in a streaming * fashion */ public BlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address, OutStreamOptions options) throws IOException { if (blockSize == -1) { try (CloseableResource<BlockMasterClient> blockMasterClientResource = mContext.acquireBlockMasterClientResource()) { blockSize = blockMasterClientResource.get().getBlockInfo(blockId).getLength(); } } // No specified location to write to. if (address == null) { throw new ResourceExhaustedException(ExceptionMessage.NO_SPACE_FOR_BLOCK_ON_WORKER .getMessage(FormatUtils.getSizeFromBytes(blockSize))); } LOG.debug("Create block outstream for {} of block size {} at address {}, using options: {}", blockId, blockSize, address, options); return BlockOutStream.create(mContext, blockId, blockSize, address, options); }
/** * Gets the in-memory percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in memory percentage */ private int getInMemoryPercentage(Inode inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = inode.asFile(); long length = inodeFile.getLength(); if (length == 0) { return 100; } long inMemoryLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (isInTopStorageTier(info)) { inMemoryLength += info.getLength(); } } return (int) (inMemoryLength * 100 / length); }
/** * Check if the two BlockInfo object are equal. * * @param a the first BlockInfo object to be checked * @param b the second BlockInfo object to be checked */ public void checkEquality(BlockInfo a, BlockInfo b) { Assert.assertEquals(a.getBlockId(), b.getBlockId()); Assert.assertEquals(a.getLength(), b.getLength()); Assert.assertEquals(a.getLocations(), b.getLocations()); Assert.assertEquals(a, b); }
for (int i = 0; i < blockInfoList.size(); i++) { BlockInfo blockInfo = blockInfoList.get(i); inAlluxioLength += blockInfo.getLength(); if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) { throw new BlockInfoException( "Block index " + i + " has a block size smaller than the file block size (" + fileInode
long blockSize = info.getLength();
/** * @param blockId id of the block * @return a {@link Protocol.OpenUfsBlockOptions} based on the block id and options */ public Protocol.OpenUfsBlockOptions getOpenUfsBlockOptions(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); boolean readFromUfs = mStatus.isPersisted(); // In case it is possible to fallback to read UFS blocks, also fill in the options. boolean storedAsUfsBlock = mStatus.getPersistenceState().equals("TO_BE_PERSISTED"); readFromUfs = readFromUfs || storedAsUfsBlock; if (!readFromUfs) { return Protocol.OpenUfsBlockOptions.getDefaultInstance(); } long blockStart = BlockId.getSequenceNumber(blockId) * mStatus.getBlockSizeBytes(); BlockInfo info = getBlockInfo(blockId); Protocol.OpenUfsBlockOptions openUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder() .setUfsPath(mStatus.getUfsPath()).setOffsetInFile(blockStart).setBlockSize(info.getLength()) .setMaxUfsReadConcurrency(mProtoOptions.getMaxUfsReadConcurrency()) .setNoCache(!ReadType.fromProto(mProtoOptions.getReadType()).isCache()) .setMountId(mStatus.getMountId()).build(); if (storedAsUfsBlock) { // On client-side, we do not have enough mount information to fill in the UFS file path. // Instead, we unset the ufsPath field and fill in a flag ufsBlock to indicate the UFS file // path can be derived from mount id and the block ID. Also because the entire file is only // one block, we set the offset in file to be zero. openUfsBlockOptions = openUfsBlockOptions.toBuilder().clearUfsPath().setBlockInUfsTier(true) .setOffsetInFile(0).build(); } return openUfsBlockOptions; }
long blockLength = mOptions.getBlockInfo(blockId).getLength(); AsyncCacheRequest request = AsyncCacheRequest.newBuilder().setBlockId(blockId).setLength(blockLength)
@Override public long getLength() throws IOException { return mBlockStore.getInfo(mBlockId).getLength(); }
for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); String[] hosts = addresses.stream().map(HostAndPort::getHostText).toArray(String[]::new); blockLocations.add( new BlockLocation(names, hosts, offset, fileBlockInfo.getBlockInfo().getLength()));
/** * Creates a new instance of {@link UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
/** * Gets the in-Alluxio percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in Alluxio, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in alluxio percentage */ private int getInAlluxioPercentage(Inode<?> inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = (InodeFile) inode; long length = inodeFile.getLength(); if (length == 0) { return 100; } long inAlluxioLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (!info.getLocations().isEmpty()) { inAlluxioLength += info.getLength(); } } return (int) (inAlluxioLength * 100 / length); }
/** * Gets the in-memory percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in memory percentage */ private int getInMemoryPercentage(Inode<?> inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = (InodeFile) inode; long length = inodeFile.getLength(); if (length == 0) { return 100; } long inMemoryLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (isInTopStorageTier(info)) { inMemoryLength += info.getLength(); } } return (int) (inMemoryLength * 100 / length); }
BlockMasterClient blockMasterClient = mContext.acquireMasterClient(); try { blockSize = blockMasterClient.getBlockInfo(blockId).getLength(); } catch (AlluxioException e) { throw new IOException(e);