/** * Instantiates a new Node info. * * @param workerInfo the worker info */ public NodeInfo(WorkerInfo workerInfo) { mHost = workerInfo.getAddress().getHost(); mWebPort = workerInfo.getAddress().getWebPort(); mLastContactSec = Integer.toString(workerInfo.getLastContactSec()); mWorkerState = workerInfo.getState(); mCapacityBytes = workerInfo.getCapacityBytes(); mUsedBytes = workerInfo.getUsedBytes(); if (mCapacityBytes != 0) { mUsedPercent = (int) (100L * mUsedBytes / mCapacityBytes); } else { mUsedPercent = 0; } mFreePercent = 100 - mUsedPercent; mUptimeClockTime = WebUtils.convertMsToShortClockTime( System.currentTimeMillis() - workerInfo.getStartTimeMs()); mWorkerId = workerInfo.getId(); }
/** * Converts a proto type to a wire type. * * @param workerInfo the proto type to convert * @return the converted wire type */ public static WorkerInfo fromProto(alluxio.grpc.WorkerInfo workerInfo) { return new WorkerInfo().setAddress(fromProto(workerInfo.getAddress())) .setCapacityBytes(workerInfo.getCapacityBytes()) .setCapacityBytesOnTiers(workerInfo.getCapacityBytesOnTiers()).setId(workerInfo.getId()) .setLastContactSec(workerInfo.getLastContactSec()) .setStartTimeMs(workerInfo.getStartTimeMs()).setState(workerInfo.getState()) .setUsedBytes(workerInfo.getUsedBytes()) .setUsedBytesOnTiers(workerInfo.getUsedBytesOnTiersMap()); }
/** * @return generated {@link WorkerInfo} for this worker */ public synchronized WorkerInfo generateClientWorkerInfo() { return new WorkerInfo().setId(mId).setAddress(mWorkerAddress).setLastContactSec( (int) ((CommonUtils.getCurrentMs() - mLastUpdatedTimeMs) / Constants.SECOND_MS)) .setState("In Service").setStartTimeMs(mStartTimeMs); }
/** * Converts wire type to proto type. * * @param workerInfo the wire representation to convert * @return the converted proto representation */ public static alluxio.grpc.WorkerInfo toProto(WorkerInfo workerInfo) { return alluxio.grpc.WorkerInfo.newBuilder().setId(workerInfo.getId()) .setAddress(toProto(workerInfo.getAddress())) .setLastContactSec(workerInfo.getLastContactSec()).setState(workerInfo.getState()) .setCapacityBytes(workerInfo.getCapacityBytes()).setUsedBytes(workerInfo.getUsedBytes()) .setStartTimeMs(workerInfo.getStartTimeMs()) .putAllCapacityBytesOnTiers(workerInfo.getCapacityBytesOnTiers()) .putAllUsedBytesOnTiers(workerInfo.getUsedBytesOnTiers()).build(); }
/** * Prints worker information when only one tier exists. * * @param workerInfoList the worker info list to get info from */ private void printShortWorkerInfo(List<WorkerInfo> workerInfoList) { String tier = mCapacityTierInfoMap.firstKey(); String shortInfoFormat = getInfoFormat(workerInfoList, true); print(String.format("%n" + shortInfoFormat, "Worker Name", "Last Heartbeat", "Storage", tier)); for (WorkerInfo info : workerInfoList) { long capacityBytes = info.getCapacityBytes(); long usedBytes = info.getUsedBytes(); String usedPercentageInfo = ""; if (capacityBytes != 0) { int usedPercentage = (int) (100L * usedBytes / capacityBytes); usedPercentageInfo = String.format(" (%s%%)", usedPercentage); } print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getLastContactSec(), "capacity", FormatUtils.getSizeFromBytes(capacityBytes))); print(String.format(shortInfoFormat, "", "", "used", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo)); } }
initVariables(); for (WorkerInfo workerInfo : workerInfoList) { long usedBytes = workerInfo.getUsedBytes(); long capacityBytes = workerInfo.getCapacityBytes(); mSumCapacityBytes += capacityBytes; mSumUsedBytes += usedBytes; String workerName = workerInfo.getAddress().getHost(); Map<String, Long> totalBytesOnTiers = workerInfo.getCapacityBytesOnTiers(); for (Map.Entry<String, Long> totalBytesTier : totalBytesOnTiers.entrySet()) { String tier = totalBytesTier.getKey(); Map<String, Long> usedBytesOnTiers = workerInfo.getUsedBytesOnTiers(); for (Map.Entry<String, Long> usedBytesTier: usedBytesOnTiers.entrySet()) { String tier = usedBytesTier.getKey();
/** * @return the info of all block workers */ public List<BlockWorkerInfo> getAllWorkers() throws IOException { try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireBlockMasterClientResource()) { return masterClientResource.get().getWorkerInfoList().stream() .map(w -> new BlockWorkerInfo(w.getAddress(), w.getCapacityBytes(), w.getUsedBytes())) .collect(toList()); } }
/** * Gets the info format according to the longest worker name. * @param workerInfoList the worker info list to get info from * @param isShort whether exists only one tier * @return the info format for printing long/short worker info */ private String getInfoFormat(List<WorkerInfo> workerInfoList, boolean isShort) { int maxWorkerNameLength = workerInfoList.stream().map(w -> w.getAddress().getHost().length()) .max(Comparator.comparing(Integer::intValue)).get(); int firstIndent = 16; if (firstIndent <= maxWorkerNameLength) { // extend first indent according to the longest worker name by default 5 firstIndent = maxWorkerNameLength + 5; } if (isShort) { return "%-" + firstIndent + "s %-16s %-13s %s"; } return "%-" + firstIndent + "s %-16s %-13s %-16s %s"; }
@Test public void getOutStreamWithReplicated() throws Exception { File file = File.createTempFile("test", ".tmp"); CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder() .setPath(file.getAbsolutePath()).build(); when(mWorkerClient.createLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<CreateLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(response); return mStreamObserver; } }); when(mMasterClient.getWorkerInfoList()).thenReturn(Lists .newArrayList(new alluxio.wire.WorkerInfo().setAddress(WORKER_NET_ADDRESS_LOCAL), new alluxio.wire.WorkerInfo().setAddress(WORKER_NET_ADDRESS_REMOTE))); OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setLocationPolicy(new MockFileWriteLocationPolicy( Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL, WORKER_NET_ADDRESS_REMOTE))) .setWriteType(WriteType.MUST_CACHE).setReplicationMin(2); BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); assertEquals(alluxio.client.block.stream.BlockOutStream.class, stream.getClass()); }
/** * Cancels the current job. */ public synchronized void cancel() { for (int taskId : mJobInfo.getTaskIdList()) { mCommandManager.submitCancelTaskCommand(mJobInfo.getId(), taskId, mTaskIdToWorkerInfo.get(taskId).getId()); } }
@Before public void before() throws Exception { mCommandManager = new CommandManager(); // Create mock job info. mJobconfig = Mockito.mock(JobConfig.class, Mockito.withSettings().serializable()); Mockito.when(mJobconfig.getName()).thenReturn("mock"); mJobId = 1; // Create mock job definition. @SuppressWarnings("unchecked") JobDefinition<JobConfig, Serializable, Serializable> mockJobDefinition = Mockito.mock(JobDefinition.class); JobDefinitionRegistry singleton = PowerMockito.mock(JobDefinitionRegistry.class); Whitebox.setInternalState(JobDefinitionRegistry.class, "INSTANCE", singleton); Mockito.when(singleton.getJobDefinition(mJobconfig)).thenReturn(mockJobDefinition); mJobDefinition = mockJobDefinition; // Create test worker. mWorkerInfo = new WorkerInfo(); mWorkerInfo.setId(0); mWorkerInfoList = Lists.newArrayList(mWorkerInfo); mUfsManager = Mockito.mock(UfsManager.class); }
@Test public void workerHeartbeatUpdatesMemoryCount() throws Exception { // Create a worker. long worker = mBlockMaster.getWorkerId(NET_ADDRESS_1); Map<String, Long> initialUsedBytesOnTiers = ImmutableMap.of("MEM", 50L); mBlockMaster.workerRegister(worker, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), initialUsedBytesOnTiers, NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); // Update used bytes with a worker heartbeat. Map<String, Long> newUsedBytesOnTiers = ImmutableMap.of("MEM", 50L); mBlockMaster.workerHeartbeat(worker, null, newUsedBytesOnTiers, NO_BLOCKS, NO_BLOCKS_ON_TIERS, mMetrics); WorkerInfo workerInfo = Iterables.getOnlyElement(mBlockMaster.getWorkerInfoList()); assertEquals(50, workerInfo.getUsedBytes()); }
@Override public int compare(WorkerInfo o1, WorkerInfo o2) { return o1.getLastContactSec() - o2.getLastContactSec(); }
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }
public void checkEquality(WorkerInfo a, WorkerInfo b) { Assert.assertEquals(a.getId(), b.getId()); Assert.assertEquals(a.getAddress(), b.getAddress()); Assert.assertEquals(a.getLastContactSec(), b.getLastContactSec()); Assert.assertEquals(a.getCapacityBytes(), b.getCapacityBytes()); Assert.assertEquals(a.getUsedBytes(), b.getUsedBytes()); Assert.assertEquals(a.getStartTimeMs(), b.getStartTimeMs()); Assert.assertEquals(a.getState(), b.getState()); Assert.assertEquals(a.getCapacityBytesOnTiers(), b.getCapacityBytesOnTiers()); Assert.assertEquals(a.getUsedBytesOnTiers(), b.getUsedBytesOnTiers()); Assert.assertEquals(a, b); }
String workerName = info.getAddress().getHost(); long usedBytes = info.getUsedBytes(); long capacityBytes = info.getCapacityBytes(); String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName); print(String.format(longInfoFormat, workerName, info.getLastContactSec(), "capacity", FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo)); print(String.format(longInfoFormat, "", "", "used",
/** * @return the info of all active block workers * @throws IOException when work info list cannot be obtained from master * @throws AlluxioException if network connection failed */ public List<BlockWorkerInfo> getWorkerInfoList() throws IOException, AlluxioException { List<BlockWorkerInfo> infoList = Lists.newArrayList(); BlockMasterClient masterClient = mContext.acquireMasterClient(); try { for (WorkerInfo workerInfo : masterClient.getWorkerInfoList()) { infoList.add(new BlockWorkerInfo(workerInfo.getAddress(), workerInfo.getCapacityBytes(), workerInfo.getUsedBytes())); } return infoList; } finally { mContext.releaseMasterClient(masterClient); } }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); Preconditions.checkArgument(numReplicas > 0); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that don't have this block locally to replicate if (!hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }