private List<FileBlockInfo> getFileBlocks(AlluxioURI path) throws IOException { try { return mFileSystem.getStatus(path).getFileBlockInfos(); } catch (AlluxioException e) { throw new IOException(e); } }
/** * @param blockId id of the block * @return the block info associated with the block id, note that this will be a cached copy * and will not fetch the latest info from the master */ public BlockInfo getBlockInfo(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); return mStatus.getFileBlockInfos().stream().map(FileBlockInfo::getBlockInfo) .filter(blockInfo -> blockInfo.getBlockId() == blockId).findFirst().get(); }
private WorkerInfo getBestJobWorker(URIStatus status, List<BlockWorkerInfo> alluxioWorkerInfoList, List<WorkerInfo> jobWorkerInfoList, Map<String, WorkerInfo> hostnameToJobWorker) { BlockWorkerInfo bestWorker = JobUtils.getWorkerWithMostBlocks(alluxioWorkerInfoList, status.getFileBlockInfos()); if (bestWorker == null) { // Nobody has blocks, choose a random worker. bestWorker = alluxioWorkerInfoList.get(mRandom.nextInt(jobWorkerInfoList.size())); } // Map the best Alluxio worker to a job worker. WorkerInfo worker = hostnameToJobWorker.get(bestWorker.getNetAddress().getHost()); if (worker == null) { return jobWorkerInfoList.get(new Random().nextInt(jobWorkerInfoList.size())); } return worker; }
switch (formatSpecifier) { case 'b': resp = status.isFolder() ? "NA" : String.valueOf(status.getFileBlockInfos().size()); break; case 'g':
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(PersistConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { if (jobWorkerInfoList.isEmpty()) { throw new RuntimeException("No worker is available"); } AlluxioURI uri = new AlluxioURI(config.getFilePath()); List<BlockWorkerInfo> alluxioWorkerInfoList = AlluxioBlockStore.create(mFsContext).getAllWorkers(); BlockWorkerInfo workerWithMostBlocks = JobUtils.getWorkerWithMostBlocks(alluxioWorkerInfoList, mFileSystem.getStatus(uri).getFileBlockInfos()); // Map the best Alluxio worker to a job worker. Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); boolean found = false; if (workerWithMostBlocks != null) { for (WorkerInfo workerInfo : jobWorkerInfoList) { if (workerInfo.getAddress().getHost() .equals(workerWithMostBlocks.getNetAddress().getHost())) { result.put(workerInfo, null); found = true; break; } } } if (!found) { result.put(jobWorkerInfoList.get(new Random().nextInt(jobWorkerInfoList.size())), null); } return result; }
for (FileBlockInfo blockInfo : mFileSystem.getStatus(uri).getFileBlockInfos()) { List<String> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo); int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size();
private void triggerAsyncCaching(BlockInStream stream) throws IOException { boolean cache = ReadType.fromProto(mOptions.getOptions().getReadType()).isCache(); boolean overReplicated = mStatus.getReplicationMax() > 0 && mStatus.getFileBlockInfos().get((int) (getPos() / mBlockSize)) .getBlockInfo().getLocations().size() >= mStatus.getReplicationMax(); cache = cache && !overReplicated;
Assert.assertEquals(uriStatus.isPinned(), fileInfo.isPinned()); Assert.assertEquals(uriStatus.isMountPoint(), fileInfo.isMountPoint()); Assert.assertEquals(uriStatus.getFileBlockInfos(), fileInfo.getFileBlockInfos()); Assert.assertEquals(uriStatus.toString(), fileInfo.toString());
private List<FileBlockInfo> getFileBlocks(AlluxioURI path) throws IOException { try { return mFileSystem.getStatus(path).getFileBlockInfos(); } catch (AlluxioException e) { throw new IOException(e); } }