/** * @param blockWorkers a list of block workers * @param blockInfo information about a block * @return the block worker hosts which are not storing the specified block */ private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers, FileBlockInfo blockInfo) { List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream() .map(location -> location.getWorkerAddress().getHost()) .collect(Collectors.toList()); return blockWorkers.stream() .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost())) .map(worker -> worker.getNetAddress().getHost()) .collect(Collectors.toList()); }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); Preconditions.checkArgument(numReplicas > 0); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that don't have this block locally to replicate if (!hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(EvictConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that have this block locally to evict if (hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { BlockWorkerInfo worker = addressIndexedWorkers.getFirstByField(WORKER_ADDRESS_INDEX, location.getWorkerAddress()); if (worker == null) {
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); System.out.println(plainPath + " with file id " + status.getFileId() + " is on nodes: "); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); for (long blockId : status.getBlockIds()) { for (BlockLocation location : blockStore.getInfo(blockId).getLocations()) { System.out.println(location.getWorkerAddress().getHost()); } } }
WorkerNetAddress dataSource = null; locations = locations.stream() .filter(location -> workers.contains(location.getWorkerAddress())).collect(toList()); locations.stream().map(location -> location.getWorkerAddress().getTieredIdentity()) .collect(toList()); Collections.shuffle(tieredLocations);
/** * Converts wire type to proto type. * * @param blockLocation the wire type to convert * @return the converted proto type */ public static alluxio.grpc.BlockLocation toProto(BlockLocation blockLocation) { return alluxio.grpc.BlockLocation.newBuilder().setWorkerId(blockLocation.getWorkerId()) .setWorkerAddress(toProto(blockLocation.getWorkerAddress())) .setTierAlias(blockLocation.getTierAlias()).build(); }
public void checkEquality(BlockLocation a, BlockLocation b) { Assert.assertEquals(a.getWorkerId(), b.getWorkerId()); Assert.assertEquals(a.getWorkerAddress(), b.getWorkerAddress()); Assert.assertEquals(a.getTierAlias(), b.getTierAlias()); Assert.assertEquals(a, b); }
WorkerNetAddress address = location.getWorkerAddress(); locations.add(address.getHost() + ":" + address.getDataPort());
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHostText()); } mLocations.addAll(locations); }
WorkerNetAddress workerNetAddress = location.getWorkerAddress(); if (workerNetAddress.getHost().equals(localHostName)) { WorkerNetAddress workerNetAddress = blockInfo.getLocations().get(0).getWorkerAddress(); return new RemoteBlockInStream(blockId, blockInfo.getLength(), workerNetAddress);
addrs.add(location.getWorkerAddress());
WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress(); BlockWorkerClient blockWorkerClient = mContext.acquireWorkerClient(workerAddr.getHost()); try {
WorkerNetAddress address = location.getWorkerAddress(); locations.add(address.getHost() + ":" + address.getDataPort());
BlockInfo blockInfo = mContext.getAluxioBlockStore().getInfo(currentBlockId); for (BlockLocation location : blockInfo.getLocations()) { if (address.equals(location.getWorkerAddress())) { mShouldCacheCurrentBlock = false;