private void init() { if (mInputStream != null) { return; } WorkerNetAddress address = new WorkerNetAddress().setHost(mDataSource.getHostName()) .setDataPort(mDataSource.getPort()); mInputStream = BlockInStream.createRemoteBlockInStream(mFsContext, mBlockId, address, BlockInStream.BlockInStreamSource.REMOTE, mBlockSize, mUfsOptions); mChannel = Channels.newChannel(mInputStream); } }
@Test public void getBlockLocationsOnlyInAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoMatchingWorkersDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoUfsLocationsDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsInUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsOnlyMatchingWorkers() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", worker2.getHost(), "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsInUfsAndAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
private BlockWorkerInfo worker(long capacity, long used, String node, String rack) { WorkerNetAddress address = new WorkerNetAddress(); List<LocalityTier> tiers = new ArrayList<>(); if (node != null && !node.isEmpty()) { address.setHost(node); tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); } if (rack != null && !rack.isEmpty()) { tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); } address.setTieredIdentity(new TieredIdentity(tiers)); return new BlockWorkerInfo(address, capacity, used); } }
private BlockWorkerInfo worker(long capacity, String node, String rack) { WorkerNetAddress address = new WorkerNetAddress(); List<LocalityTier> tiers = new ArrayList<>(); if (node != null && !node.isEmpty()) { address.setHost(node); tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); } if (rack != null && !rack.isEmpty()) { tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); } address.setTieredIdentity(new TieredIdentity(tiers)); return new BlockWorkerInfo(address, capacity, 0); } }
@Test public void getBlockLocationsNoMatchingWorkersWithFallback() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1, worker2); try (Closeable conf = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED, "true", mConfiguration) .toResource()) { verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); } }
@Test public void getBlockLocationsNoUfsLocationsWithFallback() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1, worker2); try (Closeable conf = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED, "true", mConfiguration) .toResource()) { verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); } }
public static WorkerNetAddress createRandom() { WorkerNetAddress result = new WorkerNetAddress(); Random random = new Random(); String host = CommonUtils.randomAlphaNumString(random.nextInt(10)); int rpcPort = random.nextInt(); int dataPort = random.nextInt(); int webPort = random.nextInt(); TieredIdentity identity = TieredIdentityTest.createRandomTieredIdentity(); result.setHost(host); result.setRpcPort(rpcPort); result.setDataPort(dataPort); result.setWebPort(webPort); result.setTieredIdentity(identity); return result; } }
@Test public void skipJobWorkersWithoutLocalBlockWorkers() throws Exception { List<BlockWorkerInfo> blockWorkers = Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress().setHost("host0"), 0, 0)); Mockito.when(mMockBlockStore.getAllWorkers()).thenReturn(blockWorkers); createFileWithNoLocations(TEST_URI, 10); LoadConfig config = new LoadConfig(TEST_URI, 1); Map<WorkerInfo, ArrayList<LoadTask>> assignments = new LoadDefinition(mMockFsContext, mMockFileSystem).selectExecutors(config, JOB_WORKERS, mMockJobMasterContext); Assert.assertEquals(1, assignments.size()); Assert.assertEquals(10, assignments.values().iterator().next().size()); }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, ServerConfiguration.global())) .setRpcPort(mRpcAddress.getPort()) .setDataPort(getDataLocalPort()) .setDomainSocketPath(getDataDomainSocketPath()) .setWebPort(mWebServer.getLocalPort()) .setTieredIdentity(mTieredIdentitiy); }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.JOB_WORKER_RPC, ServerConfiguration.global())) .setRpcPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_RPC_PORT)) .setDataPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_DATA_PORT)) .setWebPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_WEB_PORT)); }
/** * Tests the {@link NetworkAddressUtils#getFqdnHost(InetSocketAddress)} and * {@link NetworkAddressUtils#getFqdnHost(WorkerNetAddress)} methods. */ @Test public void getFqdnHost() throws UnknownHostException { assertEquals(NetworkAddressUtils.getFqdnHost(new InetSocketAddress("localhost", 0)), "localhost"); assertEquals( NetworkAddressUtils.getFqdnHost(new WorkerNetAddress().setHost("localhost")), "localhost"); }
/** * Tests that no worker is chosen when the worker specified in the policy is not part of the * worker list. */ @Test public void noMatchingHost() { SpecificHostPolicy policy = new SpecificHostPolicy("worker3"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertNull(policy.getWorkerForNextBlock(workerInfoList, Constants.MB)); } }
/** * Tests that the correct worker is returned when using the policy. */ @Test public void policy() { SpecificHostPolicy policy = new SpecificHostPolicy("worker2"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertEquals("worker2", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }
@Test public void getOutStreamRemote() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setLocationPolicy(new MockFileWriteLocationPolicy(Arrays.asList(worker1, worker2))) .setWriteType(WriteType.MUST_CACHE); BlockOutStream stream1 = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); assertEquals(worker1, stream1.getAddress()); BlockOutStream stream2 = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); assertEquals(worker2, stream2.getAddress()); }
/** * Converts a proto type to a wire type. * * @param workerNetPAddress the proto type to convert * @return the converted wire type */ public static WorkerNetAddress fromProto(alluxio.grpc.WorkerNetAddress workerNetPAddress) { WorkerNetAddress workerNetAddress = new WorkerNetAddress(); workerNetAddress.setHost(workerNetPAddress.getHost()); workerNetAddress.setRpcPort(workerNetPAddress.getRpcPort()); workerNetAddress.setDataPort(workerNetPAddress.getDataPort()); workerNetAddress.setWebPort(workerNetPAddress.getWebPort()); workerNetAddress.setDomainSocketPath(workerNetPAddress.getDomainSocketPath()); workerNetAddress.setTieredIdentity(fromProto(workerNetPAddress.getTieredIdentity())); return workerNetAddress; }