public static WorkerNetAddress createRandom() { WorkerNetAddress result = new WorkerNetAddress(); Random random = new Random(); String host = CommonUtils.randomAlphaNumString(random.nextInt(10)); int rpcPort = random.nextInt(); int dataPort = random.nextInt(); int webPort = random.nextInt(); TieredIdentity identity = TieredIdentityTest.createRandomTieredIdentity(); result.setHost(host); result.setRpcPort(rpcPort); result.setDataPort(dataPort); result.setWebPort(webPort); result.setTieredIdentity(identity); return result; } }
/** * Helper to register a new worker. * * @param workerIndex the index of the worker in all workers * @return the created worker ID */ private long createWorkerHelper(int workerIndex) throws Exception { WorkerNetAddress address = new WorkerNetAddress().setHost("host" + workerIndex).setRpcPort(1000) .setDataPort(2000).setWebPort(3000); long workerId = mBlockMaster.getWorkerId(address); if (!mKnownWorkers.contains(workerId)) { // Do not re-register works, otherwise added block will be removed mBlockMaster.workerRegister(workerId, ImmutableList.of("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mKnownWorkers.add(workerId); } return workerId; }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, ServerConfiguration.global())) .setRpcPort(mRpcAddress.getPort()) .setDataPort(getDataLocalPort()) .setDomainSocketPath(getDataDomainSocketPath()) .setWebPort(mWebServer.getLocalPort()) .setTieredIdentity(mTieredIdentitiy); }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.JOB_WORKER_RPC, ServerConfiguration.global())) .setRpcPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_RPC_PORT)) .setDataPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_DATA_PORT)) .setWebPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_WEB_PORT)); }
@Before public void before() { mWorkerInfos.clear(); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 2 * (long) Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 3 * (long) Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker4").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 3 * (long) Constants.GB, 0)); }
/** * Tests that no worker is chosen when the worker specified in the policy is not part of the * worker list. */ @Test public void noMatchingHost() { SpecificHostPolicy policy = new SpecificHostPolicy("worker3"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertNull(policy.getWorkerForNextBlock(workerInfoList, Constants.MB)); } }
/** * Tests for setBlockWorkerInfo. */ @Test public void setBlockWorkerInfoTest() { mWorkerInfos.clear(); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 2 * (long) Constants.GB, 0)); GetWorkerOptions options = GetWorkerOptions.defaults(); options.setBlockWorkerInfos(mWorkerInfos); assertEquals(mWorkerInfos, options.getBlockWorkerInfos()); }
/** * Tests that the correct worker is returned when using the policy. */ @Test public void policy() { SpecificHostPolicy policy = new SpecificHostPolicy("worker2"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertEquals("worker2", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }
/** * Tests that the worker with the most available space is chosen. */ @Test public void getMostAvailableWorker() { List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0)); MostAvailableFirstPolicy policy = new MostAvailableFirstPolicy(ConfigurationTestUtils .defaults()); Assert.assertEquals("worker3", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }
/** * Tests that the correct workers are chosen when round robin is used. */ @Test public void getWorker() { List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0)); RoundRobinPolicy policy = new RoundRobinPolicy(ConfigurationTestUtils.defaults()); Assert.assertNotEquals( policy.getWorkerForNextBlock(workerInfoList, 2 * (long) Constants.GB).getHost(), policy.getWorkerForNextBlock(workerInfoList, 2 * (long) Constants.GB).getHost()); }
/** * Converts a proto type to a wire type. * * @param workerNetPAddress the proto type to convert * @return the converted wire type */ public static WorkerNetAddress fromProto(alluxio.grpc.WorkerNetAddress workerNetPAddress) { WorkerNetAddress workerNetAddress = new WorkerNetAddress(); workerNetAddress.setHost(workerNetPAddress.getHost()); workerNetAddress.setRpcPort(workerNetPAddress.getRpcPort()); workerNetAddress.setDataPort(workerNetPAddress.getDataPort()); workerNetAddress.setWebPort(workerNetPAddress.getWebPort()); workerNetAddress.setDomainSocketPath(workerNetPAddress.getDomainSocketPath()); workerNetAddress.setTieredIdentity(fromProto(workerNetPAddress.getTieredIdentity())); return workerNetAddress; }
@Test public void heartbeatFileUnderReplicatedAndLost() throws Exception { mFileContext.getOptions().setReplicationMin(2); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); // Create a worker. long workerId = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost") .setRpcPort(80).setDataPort(81).setWebPort(82)); mBlockMaster.workerRegister(workerId, Collections.singletonList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(workerId, 50L, "MEM", blockId, 20L); // Indicate that blockId is removed on the worker. mBlockMaster.workerHeartbeat(workerId, null, ImmutableMap.of("MEM", 0L), ImmutableList.of(blockId), NO_BLOCKS_ON_TIERS, NO_METRICS); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); }
new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost") .setTieredIdentity(TieredIdentityFactory.fromString("node=localhost", sConf)) .setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0); when(mBlockStore.getEligibleWorkers()).thenReturn(Lists.newArrayList(workerInfo)); mAlluxioOutStreamMap = outStreamMap;
new WorkerNetAddress().setHost("localhost").setRpcPort(80).setDataPort(81).setWebPort(82)); mBlockMaster.workerRegister(mWorkerId1, Arrays.asList("MEM", "SSD"), ImmutableMap.of("MEM", (long) Constants.MB, "SSD", (long) Constants.MB), new HashMap<String, List<Long>>(), RegisterWorkerPOptions.getDefaultInstance()); mWorkerId2 = mBlockMaster.getWorkerId( new WorkerNetAddress().setHost("remote").setRpcPort(80).setDataPort(81).setWebPort(82)); mBlockMaster.workerRegister(mWorkerId2, Arrays.asList("MEM", "SSD"), ImmutableMap.of("MEM", (long) Constants.MB, "SSD", (long) Constants.MB),
/** * Creates a new instance of {@link WorkerNetAddress} from thrift representation. * * @param address the thrift net address * @return the instance */ public static WorkerNetAddress fromThrift(alluxio.thrift.WorkerNetAddress address) { TieredIdentity tieredIdentity = TieredIdentity.fromThrift(address.getTieredIdentity()); if (tieredIdentity == null) { // This means the worker is pre-1.7.0. We handle this in post-1.7.0 clients by filling out // the tiered identity using the hostname field. tieredIdentity = new TieredIdentity( Arrays.asList(new LocalityTier(Constants.LOCALITY_NODE, address.getHost()))); } return new WorkerNetAddress() .setDataPort(address.getDataPort()) .setDomainSocketPath(address.getDomainSocketPath()) .setHost(address.getHost()) .setRpcPort(address.getRpcPort()) .setTieredIdentity(tieredIdentity) .setWebPort(address.getWebPort()); }