private void init() { if (mInputStream != null) { return; } WorkerNetAddress address = new WorkerNetAddress().setHost(mDataSource.getHostName()) .setDataPort(mDataSource.getPort()); mInputStream = BlockInStream.createRemoteBlockInStream(mFsContext, mBlockId, address, BlockInStream.BlockInStreamSource.REMOTE, mBlockSize, mUfsOptions); mChannel = Channels.newChannel(mInputStream); } }
@Test public void getBlockLocationsOnlyInAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoUfsLocationsDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoMatchingWorkersDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsInUfsAndAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsOnlyMatchingWorkers() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", worker2.getHost(), "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsInUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoUfsLocationsWithFallback() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1, worker2); try (Closeable conf = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED, "true", mConfiguration) .toResource()) { verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); } }
@Test public void getBlockLocationsNoMatchingWorkersWithFallback() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1, worker2); try (Closeable conf = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED, "true", mConfiguration) .toResource()) { verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); } }
public static WorkerNetAddress createRandom() { WorkerNetAddress result = new WorkerNetAddress(); Random random = new Random(); String host = CommonUtils.randomAlphaNumString(random.nextInt(10)); int rpcPort = random.nextInt(); int dataPort = random.nextInt(); int webPort = random.nextInt(); TieredIdentity identity = TieredIdentityTest.createRandomTieredIdentity(); result.setHost(host); result.setRpcPort(rpcPort); result.setDataPort(dataPort); result.setWebPort(webPort); result.setTieredIdentity(identity); return result; } }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, ServerConfiguration.global())) .setRpcPort(mRpcAddress.getPort()) .setDataPort(getDataLocalPort()) .setDomainSocketPath(getDataDomainSocketPath()) .setWebPort(mWebServer.getLocalPort()) .setTieredIdentity(mTieredIdentitiy); }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.JOB_WORKER_RPC, ServerConfiguration.global())) .setRpcPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_RPC_PORT)) .setDataPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_DATA_PORT)) .setWebPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_WEB_PORT)); }
@Before public void before() { mWorkerInfos.clear(); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 2 * (long) Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 3 * (long) Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker4").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 3 * (long) Constants.GB, 0)); }
/** * Tests that no worker is chosen when the worker specified in the policy is not part of the * worker list. */ @Test public void noMatchingHost() { SpecificHostPolicy policy = new SpecificHostPolicy("worker3"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertNull(policy.getWorkerForNextBlock(workerInfoList, Constants.MB)); } }
/** * Tests that the correct worker is returned when using the policy. */ @Test public void policy() { SpecificHostPolicy policy = new SpecificHostPolicy("worker2"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertEquals("worker2", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }
/** * Tests for setBlockWorkerInfo. */ @Test public void setBlockWorkerInfoTest() { mWorkerInfos.clear(); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), Constants.GB, 0)); mWorkerInfos.add(new BlockWorkerInfo( new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT) .setWebPort(PORT), 2 * (long) Constants.GB, 0)); GetWorkerOptions options = GetWorkerOptions.defaults(); options.setBlockWorkerInfos(mWorkerInfos); assertEquals(mWorkerInfos, options.getBlockWorkerInfos()); }
/** * Tests that the worker with the most available space is chosen. */ @Test public void getMostAvailableWorker() { List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0)); MostAvailableFirstPolicy policy = new MostAvailableFirstPolicy(ConfigurationTestUtils .defaults()); Assert.assertEquals("worker3", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }
/** * Tests that the correct workers are chosen when round robin is used. */ @Test public void getWorker() { List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0)); RoundRobinPolicy policy = new RoundRobinPolicy(ConfigurationTestUtils.defaults()); Assert.assertNotEquals( policy.getWorkerForNextBlock(workerInfoList, 2 * (long) Constants.GB).getHost(), policy.getWorkerForNextBlock(workerInfoList, 2 * (long) Constants.GB).getHost()); }
/** * Converts a proto type to a wire type. * * @param workerNetPAddress the proto type to convert * @return the converted wire type */ public static WorkerNetAddress fromProto(alluxio.grpc.WorkerNetAddress workerNetPAddress) { WorkerNetAddress workerNetAddress = new WorkerNetAddress(); workerNetAddress.setHost(workerNetPAddress.getHost()); workerNetAddress.setRpcPort(workerNetPAddress.getRpcPort()); workerNetAddress.setDataPort(workerNetPAddress.getDataPort()); workerNetAddress.setWebPort(workerNetPAddress.getWebPort()); workerNetAddress.setDomainSocketPath(workerNetPAddress.getDomainSocketPath()); workerNetAddress.setTieredIdentity(fromProto(workerNetPAddress.getTieredIdentity())); return workerNetAddress; }
@Test public void heartbeatFileUnderReplicatedAndLost() throws Exception { mFileContext.getOptions().setReplicationMin(2); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); // Create a worker. long workerId = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost") .setRpcPort(80).setDataPort(81).setWebPort(82)); mBlockMaster.workerRegister(workerId, Collections.singletonList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(workerId, 50L, "MEM", blockId, 20L); // Indicate that blockId is removed on the worker. mBlockMaster.workerHeartbeat(workerId, null, ImmutableMap.of("MEM", 0L), ImmutableList.of(blockId), NO_BLOCKS_ON_TIERS, NO_METRICS); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); }