/** * Extracts rpcPort InetSocketAddress from Alluxio representation of network address. * * @param netAddress the input network address representation * @return InetSocketAddress */ public static InetSocketAddress getRpcPortSocketAddress(WorkerNetAddress netAddress) { String host = netAddress.getHost(); int port = netAddress.getRpcPort(); return new InetSocketAddress(host, port); }
/** * Gets FQDN(Full Qualified Domain Name) from Alluxio representation of network address. * * @param addr the input network address representation * @return the resolved FQDN host name * @throws UnknownHostException if the host is not known */ public static String getFqdnHost(WorkerNetAddress addr) throws UnknownHostException { return resolveHostName(addr.getHost()); }
/** * @param address the Alluxio worker network address * @param conf Alluxio configuration * @return true if the worker is local */ public static boolean isLocalHost(WorkerNetAddress address, AlluxioConfiguration conf) { return address.getHost().equals(NetworkAddressUtils.getClientHostName(conf)); }
@Override public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) { // find the first worker matching the host name for (BlockWorkerInfo info : workerInfoList) { if (info.getNetAddress().getHost().equals(mHostname)) { return info.getNetAddress(); } } return null; }
private void initializeLocalWorker() throws IOException { List<WorkerNetAddress> addresses = getWorkerAddresses(); if (!addresses.isEmpty()) { if (addresses.get(0).getHost().equals(NetworkAddressUtils.getClientHostName(mClientContext .getConf()))) { mLocalWorker = addresses.get(0); } } mLocalWorkerInitialized = true; }
private Map<String, WorkerNetAddress> getHostToWorkerMap() throws IOException { List<BlockWorkerInfo> workers = AlluxioBlockStore.create(mFsContext).getEligibleWorkers(); return workers.stream().collect( toMap(worker -> worker.getNetAddress().getHost(), BlockWorkerInfo::getNetAddress, (worker1, worker2) -> worker1)); } }
@GuardedBy("workerInfo") private void processWorkerOrphanedBlocks(MasterWorkerInfo workerInfo) { for (long block : workerInfo.getBlocks()) { if (!mBlockStore.getBlock(block).isPresent()) { LOG.info("Requesting delete for orphaned block: {} from worker {}.", block, workerInfo.getWorkerAddress().getHost()); workerInfo.updateToRemovedBlock(true, block); } } }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); System.out.println(plainPath + " with file id " + status.getFileId() + " is on nodes: "); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); for (long blockId : status.getBlockIds()) { for (BlockLocation location : blockStore.getInfo(blockId).getLocations()) { System.out.println(location.getWorkerAddress().getHost()); } } }
@Override public String createBlock(long sessionId, long blockId, String tierAlias, long initialBytes) throws BlockAlreadyExistsException, WorkerOutOfSpaceException, IOException { BlockStoreLocation loc = BlockStoreLocation.anyDirInTier(tierAlias); TempBlockMeta createdBlock; try { createdBlock = mBlockStore.createBlock(sessionId, blockId, loc, initialBytes); } catch (WorkerOutOfSpaceException e) { InetSocketAddress address = InetSocketAddress.createUnresolved(mAddress.getHost(), mAddress.getRpcPort()); throw new WorkerOutOfSpaceException(ExceptionMessage.CANNOT_REQUEST_SPACE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, address, blockId), e); } return createdBlock.getPath(); }
@Test public void getBlockLocationsInUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
/** * Tests that another worker is picked in case the local host does not have enough capacity. */ @Test public void getOthersWhenNotEnoughCapacityOnLocal() { String localhostName = NetworkAddressUtils.getLocalHostName(sResolutionTimeout); LocalFirstPolicy policy = new LocalFirstPolicy(sConf); List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, "worker1", "")); workers.add(worker(Constants.MB, localhostName, "")); assertEquals("worker1", policy.getWorkerForNextBlock(workers, Constants.GB).getHost()); }
@Test public void getBlockLocationsInUfsAndAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsOnlyMatchingWorkers() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", worker2.getHost(), "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
/** * Tests that another worker is picked in case the local host does not have enough availability. */ @Test public void getOthersWhenNotEnoughAvailabilityOnLocal() { String localhostName = NetworkAddressUtils.getLocalHostName(1000); FileWriteLocationPolicy policy = new LocalFirstAvoidEvictionPolicy(mConf); List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, 0, "worker1", "")); workers.add(worker(Constants.MB, Constants.MB, localhostName, "")); assertEquals("worker1", policy.getWorkerForNextBlock(workers, Constants.MB).getHost()); }
/** * Tests that local host is picked if none of the workers has enough availability. */ @Test public void getLocalWhenNoneHasAvailability() { String localhostName = NetworkAddressUtils.getLocalHostName(1000); FileWriteLocationPolicy policy = new LocalFirstAvoidEvictionPolicy(mConf); List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, Constants.MB, "worker1", "")); workers.add(worker(Constants.GB, Constants.MB, localhostName, "")); assertEquals(localhostName, policy.getWorkerForNextBlock(workers, Constants.GB).getHost()); }
/** * Tests that the local host is returned first. */ @Test public void getLocalFirst() { String localhostName = NetworkAddressUtils.getLocalHostName(sResolutionTimeout); LocalFirstPolicy policy = new LocalFirstPolicy(sConf); List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, "worker1", "")); workers.add(worker(Constants.GB, localhostName, "")); assertEquals(localhostName, policy.getWorkerForNextBlock(workers, Constants.MB).getHost()); }
public void checkEquality(WorkerNetAddress a, WorkerNetAddress b) { Assert.assertEquals(a.getHost(), b.getHost()); Assert.assertEquals(a.getRpcPort(), b.getRpcPort()); Assert.assertEquals(a.getDataPort(), b.getDataPort()); Assert.assertEquals(a.getWebPort(), b.getWebPort()); Assert.assertEquals(a.getTieredIdentity(), b.getTieredIdentity()); Assert.assertEquals(a, b); }
@Test public void getWorkerEnoughCapacity() { DeterministicHashPolicy policy = (DeterministicHashPolicy) BlockLocationPolicy.Factory.create( CreateOptions.defaults() .setLocationPolicyClassName(DeterministicHashPolicy.class.getCanonicalName()), sConf); for (long blockId = 0; blockId < 100; blockId++) { // worker1 does not have enough capacity. It should never be picked. Assert.assertNotEquals("worker1", policy.getWorker( GetWorkerOptions.defaults().setBlockWorkerInfos(mWorkerInfos).setBlockId(blockId) .setBlockSize(2 * (long) Constants.GB)).getHost()); } }
/** * Tests that the correct worker is returned when using the policy. */ @Test public void policy() { SpecificHostPolicy policy = new SpecificHostPolicy("worker2"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertEquals("worker2", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }