/** * Gets the info format according to the longest worker name. * @param workerInfoList the worker info list to get info from * @param isShort whether exists only one tier * @return the info format for printing long/short worker info */ private String getInfoFormat(List<WorkerInfo> workerInfoList, boolean isShort) { int maxWorkerNameLength = workerInfoList.stream().map(w -> w.getAddress().getHost().length()) .max(Comparator.comparing(Integer::intValue)).get(); int firstIndent = 16; if (firstIndent <= maxWorkerNameLength) { // extend first indent according to the longest worker name by default 5 firstIndent = maxWorkerNameLength + 5; } if (isShort) { return "%-" + firstIndent + "s %-16s %-13s %s"; } return "%-" + firstIndent + "s %-16s %-13s %-16s %s"; }
/** * @return the info of all block workers */ public List<BlockWorkerInfo> getAllWorkers() throws IOException { try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireBlockMasterClientResource()) { return masterClientResource.get().getWorkerInfoList().stream() .map(w -> new BlockWorkerInfo(w.getAddress(), w.getCapacityBytes(), w.getUsedBytes())) .collect(toList()); } }
mSumUsedBytes += usedBytes; String workerName = workerInfo.getAddress().getHost();
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); Preconditions.checkArgument(numReplicas > 0); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that don't have this block locally to replicate if (!hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(EvictConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that have this block locally to evict if (hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
/** * @return if there are any local workers, the returned list will ONLY contain the local workers, * otherwise a list of all remote workers will be returned */ private List<WorkerNetAddress> getWorkerAddresses() throws IOException { List<WorkerInfo> infos; BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire(); try { infos = blockMasterClient.getWorkerInfoList(); } finally { mBlockMasterClientPool.release(blockMasterClient); } if (infos.isEmpty()) { throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); } // Convert the worker infos into net addresses, if there are local addresses, only keep those List<WorkerNetAddress> workerNetAddresses = new ArrayList<>(); List<WorkerNetAddress> localWorkerNetAddresses = new ArrayList<>(); String localHostname = NetworkAddressUtils.getClientHostName(mClientContext.getConf()); for (WorkerInfo info : infos) { WorkerNetAddress netAddress = info.getAddress(); if (netAddress.getHost().equals(localHostname)) { localWorkerNetAddresses.add(netAddress); } workerNetAddresses.add(netAddress); } return localWorkerNetAddresses.isEmpty() ? workerNetAddresses : localWorkerNetAddresses; }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(PersistConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { if (jobWorkerInfoList.isEmpty()) { throw new RuntimeException("No worker is available"); } AlluxioURI uri = new AlluxioURI(config.getFilePath()); List<BlockWorkerInfo> alluxioWorkerInfoList = AlluxioBlockStore.create(mFsContext).getAllWorkers(); BlockWorkerInfo workerWithMostBlocks = JobUtils.getWorkerWithMostBlocks(alluxioWorkerInfoList, mFileSystem.getStatus(uri).getFileBlockInfos()); // Map the best Alluxio worker to a job worker. Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); boolean found = false; if (workerWithMostBlocks != null) { for (WorkerInfo workerInfo : jobWorkerInfoList) { if (workerInfo.getAddress().getHost() .equals(workerWithMostBlocks.getNetAddress().getHost())) { result.put(workerInfo, null); found = true; break; } } } if (!found) { result.put(jobWorkerInfoList.get(new Random().nextInt(jobWorkerInfoList.size())), null); } return result; }
/** * Instantiates a new Node info. * * @param workerInfo the worker info */ public NodeInfo(WorkerInfo workerInfo) { mHost = workerInfo.getAddress().getHost(); mWebPort = workerInfo.getAddress().getWebPort(); mLastContactSec = Integer.toString(workerInfo.getLastContactSec()); mWorkerState = workerInfo.getState(); mCapacityBytes = workerInfo.getCapacityBytes(); mUsedBytes = workerInfo.getUsedBytes(); if (mCapacityBytes != 0) { mUsedPercent = (int) (100L * mUsedBytes / mCapacityBytes); } else { mUsedPercent = 0; } mFreePercent = 100 - mUsedPercent; mUptimeClockTime = WebUtils.convertMsToShortClockTime( System.currentTimeMillis() - workerInfo.getStartTimeMs()); mWorkerId = workerInfo.getId(); }
String workerName = info.getAddress().getHost();
/** * Prints worker information when only one tier exists. * * @param workerInfoList the worker info list to get info from */ private void printShortWorkerInfo(List<WorkerInfo> workerInfoList) { String tier = mCapacityTierInfoMap.firstKey(); String shortInfoFormat = getInfoFormat(workerInfoList, true); print(String.format("%n" + shortInfoFormat, "Worker Name", "Last Heartbeat", "Storage", tier)); for (WorkerInfo info : workerInfoList) { long capacityBytes = info.getCapacityBytes(); long usedBytes = info.getUsedBytes(); String usedPercentageInfo = ""; if (capacityBytes != 0) { int usedPercentage = (int) (100L * usedBytes / capacityBytes); usedPercentageInfo = String.format(" (%s%%)", usedPercentage); } print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getLastContactSec(), "capacity", FormatUtils.getSizeFromBytes(capacityBytes))); print(String.format(shortInfoFormat, "", "", "used", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo)); } }
List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Map<String, WorkerInfo> jobWorkersByAddress = jobWorkerInfoList.stream() .collect(Collectors.toMap(info -> info.getAddress().getHost(), info -> info));
ConcurrentMap<String, WorkerInfo> hostnameToWorker = Maps.newConcurrentMap(); for (WorkerInfo workerInfo : jobWorkerInfoList) { hostnameToWorker.put(workerInfo.getAddress().getHost(), workerInfo);
/** * Creates a file with the given name and a block on each specified worker. Workers may be * repeated to give them multiple blocks. * * @param testFile the name of the file to create * @param fileInfo file info to apply to the created file * @param workerInds the workers to put blocks on, specified by their indices * @return file info for the created file */ private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo, int... workerInds) throws Exception { AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int workerInd : workerInds) { WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress(); blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo() .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address))))); } FileInfo testFileInfo = fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; }
/** * Converts wire type to proto type. * * @param workerInfo the wire representation to convert * @return the converted proto representation */ public static alluxio.grpc.WorkerInfo toProto(WorkerInfo workerInfo) { return alluxio.grpc.WorkerInfo.newBuilder().setId(workerInfo.getId()) .setAddress(toProto(workerInfo.getAddress())) .setLastContactSec(workerInfo.getLastContactSec()).setState(workerInfo.getState()) .setCapacityBytes(workerInfo.getCapacityBytes()).setUsedBytes(workerInfo.getUsedBytes()) .setStartTimeMs(workerInfo.getStartTimeMs()) .putAllCapacityBytesOnTiers(workerInfo.getCapacityBytesOnTiers()) .putAllUsedBytesOnTiers(workerInfo.getUsedBytesOnTiers()).build(); }
public void checkEquality(WorkerInfo a, WorkerInfo b) { Assert.assertEquals(a.getId(), b.getId()); Assert.assertEquals(a.getAddress(), b.getAddress()); Assert.assertEquals(a.getLastContactSec(), b.getLastContactSec()); Assert.assertEquals(a.getCapacityBytes(), b.getCapacityBytes()); Assert.assertEquals(a.getUsedBytes(), b.getUsedBytes()); Assert.assertEquals(a.getStartTimeMs(), b.getStartTimeMs()); Assert.assertEquals(a.getState(), b.getState()); Assert.assertEquals(a.getCapacityBytesOnTiers(), b.getCapacityBytesOnTiers()); Assert.assertEquals(a.getUsedBytesOnTiers(), b.getUsedBytesOnTiers()); Assert.assertEquals(a, b); }
/** * Tests the {@link MasterWorkerInfo#generateWorkerInfo} method. */ @Test public void workerInfoGeneration() { WorkerInfo workerInfo = mInfo.generateWorkerInfo(null, true); assertEquals(mInfo.getId(), workerInfo.getId()); assertEquals(mInfo.getWorkerAddress(), workerInfo.getAddress()); assertEquals("In Service", workerInfo.getState()); assertEquals(mInfo.getCapacityBytes(), workerInfo.getCapacityBytes()); assertEquals(mInfo.getUsedBytes(), workerInfo.getUsedBytes()); assertEquals(mInfo.getStartTime(), workerInfo.getStartTimeMs()); }
/** * Gets the worker address based on its hostname by querying the master. * * @param hostname hostname of the worker to query, empty string denotes any worker * @return {@link WorkerNetAddress} of hostname, or null if no worker found */ private WorkerNetAddress getWorkerAddress(String hostname) { BlockMasterClient masterClient = acquireMasterClient(); try { List<WorkerInfo> workers = masterClient.getWorkerInfoList(); if (hostname.isEmpty() && !workers.isEmpty()) { // TODO(calvin): Do this in a more defined way. return workers.get(0).getAddress(); } for (WorkerInfo worker : workers) { if (worker.getAddress().getHost().equals(hostname)) { return worker.getAddress(); } } } catch (Exception e) { Throwables.propagate(e); } finally { releaseMasterClient(masterClient); } return null; }
private NodeInfo(WorkerInfo workerInfo) { mHost = workerInfo.getAddress().getHost(); mWebPort = workerInfo.getAddress().getWebPort(); mLastContactSec = Integer.toString(workerInfo.getLastContactSec()); mWorkerState = workerInfo.getState(); mCapacityBytes = workerInfo.getCapacityBytes(); mUsedBytes = workerInfo.getUsedBytes(); if (mCapacityBytes != 0) { mUsedPercent = (int) (100L * mUsedBytes / mCapacityBytes); } else { mUsedPercent = 0; } mFreePercent = 100 - mUsedPercent; mUptimeClockTime = WebUtils.convertMsToShortClockTime( System.currentTimeMillis() - workerInfo.getStartTimeMs()); mWorkerId = workerInfo.getId(); }
/** * @return the info of all active block workers * @throws IOException when work info list cannot be obtained from master * @throws AlluxioException if network connection failed */ public List<BlockWorkerInfo> getWorkerInfoList() throws IOException, AlluxioException { List<BlockWorkerInfo> infoList = Lists.newArrayList(); BlockMasterClient masterClient = mContext.acquireMasterClient(); try { for (WorkerInfo workerInfo : masterClient.getWorkerInfoList()) { infoList.add(new BlockWorkerInfo(workerInfo.getAddress(), workerInfo.getCapacityBytes(), workerInfo.getUsedBytes())); } return infoList; } finally { mContext.releaseMasterClient(masterClient); } }