@Override @Nullable public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) { List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList); Collections.shuffle(shuffledWorkers); // Workers must have enough capacity to hold the block. List<BlockWorkerInfo> candidateWorkers = shuffledWorkers.stream() .filter(worker -> worker.getCapacityBytes() >= blockSizeBytes) .collect(Collectors.toList()); // Try finding a worker based on nearest tiered identity. List<TieredIdentity> identities = candidateWorkers.stream() .map(worker -> worker.getNetAddress().getTieredIdentity()) .filter(Objects::nonNull) .collect(Collectors.toList()); Optional<TieredIdentity> nearest = TieredIdentityUtils.nearest(mTieredIdentity, identities, mCompareNodeIps); if (!nearest.isPresent()) { return null; } // Map back to the worker with the nearest tiered identity. return candidateWorkers.stream() .filter(worker -> worker.getNetAddress().getTieredIdentity().equals(nearest.get())) .map(worker -> worker.getNetAddress()) .findFirst().orElse(null); }
locations.stream().map(location -> location.getWorkerAddress().getTieredIdentity()) .collect(toList()); Collections.shuffle(tieredLocations); if (nearest.isPresent()) { dataSource = locations.stream().map(BlockLocation::getWorkerAddress) .filter(addr -> addr.getTieredIdentity().equals(nearest.get())).findFirst().get(); if (mTieredIdentity.getTier(0).getTierName().equals(Constants.LOCALITY_NODE) && mTieredIdentity.topTiersMatch(nearest.get())) {
/** * Converts wire type to proto type. * * @param workerNetAddress the wire representation to convert * @return the converted proto representation */ public static alluxio.grpc.WorkerNetAddress toProto(WorkerNetAddress workerNetAddress) { alluxio.grpc.WorkerNetAddress.Builder address = alluxio.grpc.WorkerNetAddress.newBuilder() .setHost(workerNetAddress.getHost()).setRpcPort(workerNetAddress.getRpcPort()) .setDataPort(workerNetAddress.getDataPort()).setWebPort(workerNetAddress.getWebPort()) .setDomainSocketPath(workerNetAddress.getDomainSocketPath()); if (workerNetAddress.getTieredIdentity() != null) { address.setTieredIdentity(toProto(workerNetAddress.getTieredIdentity())); } return address.build(); }
@Test public void chooseClosestTierAvoidEviction() throws Exception { List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, Constants.MB, "node2", "rack3")); workers.add(worker(Constants.GB, 0, "node3", "rack2")); workers.add(worker(Constants.GB, 0, "node4", "rack3")); FileWriteLocationPolicy policy; WorkerNetAddress chosen; // local rack with enough availability policy = new LocalFirstAvoidEvictionPolicy( TieredIdentityFactory.fromString("node=node2,rack=rack3", mConf), mConf); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); }
@Test public void chooseClosestTier() throws Exception { List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, "node2", "rack3")); workers.add(worker(Constants.GB, "node3", "rack2")); workers.add(worker(Constants.GB, "node4", "rack3")); LocalFirstPolicy policy; WorkerNetAddress chosen; // local rack policy = LocalFirstPolicy.create(TieredIdentityFactory.fromString("node=node1,rack=rack2", sConf), sConf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP)); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("rack2", chosen.getTieredIdentity().getTier(1).getValue()); // local node policy = LocalFirstPolicy.create(TieredIdentityFactory.fromString("node=node4,rack=rack3", sConf), sConf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP)); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); }
public void checkEquality(WorkerNetAddress a, WorkerNetAddress b) { Assert.assertEquals(a.getHost(), b.getHost()); Assert.assertEquals(a.getRpcPort(), b.getRpcPort()); Assert.assertEquals(a.getDataPort(), b.getDataPort()); Assert.assertEquals(a.getWebPort(), b.getWebPort()); Assert.assertEquals(a.getTieredIdentity(), b.getTieredIdentity()); Assert.assertEquals(a, b); }