/** * Tests that the correct worker is returned when using the policy. */ @Test public void policy() { SpecificHostPolicy policy = new SpecificHostPolicy("worker2"); List<BlockWorkerInfo> workerInfoList = new ArrayList<>(); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2") .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0)); Assert.assertEquals("worker2", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost()); }
/** * Converts wire type to proto type. * * @param workerNetAddress the wire representation to convert * @return the converted proto representation */ public static alluxio.grpc.WorkerNetAddress toProto(WorkerNetAddress workerNetAddress) { alluxio.grpc.WorkerNetAddress.Builder address = alluxio.grpc.WorkerNetAddress.newBuilder() .setHost(workerNetAddress.getHost()).setRpcPort(workerNetAddress.getRpcPort()) .setDataPort(workerNetAddress.getDataPort()).setWebPort(workerNetAddress.getWebPort()) .setDomainSocketPath(workerNetAddress.getDomainSocketPath()); if (workerNetAddress.getTieredIdentity() != null) { address.setTieredIdentity(toProto(workerNetAddress.getTieredIdentity())); } return address.build(); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof BlockLocation)) { return false; } BlockLocation that = (BlockLocation) o; return mWorkerId == that.mWorkerId && mWorkerAddress.equals(that.mWorkerAddress) && mTierAlias.equals(that.mTierAlias); }
/** * Extracts rpcPort InetSocketAddress from Alluxio representation of network address. * * @param netAddress the input network address representation * @return InetSocketAddress */ public static InetSocketAddress getRpcPortSocketAddress(WorkerNetAddress netAddress) { String host = netAddress.getHost(); int port = netAddress.getRpcPort(); return new InetSocketAddress(host, port); }
private void init() { if (mInputStream != null) { return; } WorkerNetAddress address = new WorkerNetAddress().setHost(mDataSource.getHostName()) .setDataPort(mDataSource.getPort()); mInputStream = BlockInStream.createRemoteBlockInStream(mFsContext, mBlockId, address, BlockInStream.BlockInStreamSource.REMOTE, mBlockSize, mUfsOptions); mChannel = Channels.newChannel(mInputStream); } }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, ServerConfiguration.global())) .setRpcPort(mRpcAddress.getPort()) .setDataPort(getDataLocalPort()) .setDomainSocketPath(getDataDomainSocketPath()) .setWebPort(mWebServer.getLocalPort()) .setTieredIdentity(mTieredIdentitiy); }
@Test public void getBlockLocationsInUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
/** * Gets FQDN(Full Qualified Domain Name) from Alluxio representation of network address. * * @param addr the input network address representation * @return the resolved FQDN host name * @throws UnknownHostException if the host is not known */ public static String getFqdnHost(WorkerNetAddress addr) throws UnknownHostException { return resolveHostName(addr.getHost()); }
.map(worker -> HostAndPort.fromParts(worker.getHost(), worker.getDataPort())) .collect(toList()); String[] names = addresses.stream().map(HostAndPort::toString).toArray(String[]::new);
@Test public void skipJobWorkersWithoutLocalBlockWorkers() throws Exception { List<BlockWorkerInfo> blockWorkers = Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress().setHost("host0"), 0, 0)); Mockito.when(mMockBlockStore.getAllWorkers()).thenReturn(blockWorkers); createFileWithNoLocations(TEST_URI, 10); LoadConfig config = new LoadConfig(TEST_URI, 1); Map<WorkerInfo, ArrayList<LoadTask>> assignments = new LoadDefinition(mMockFsContext, mMockFileSystem).selectExecutors(config, JOB_WORKERS, mMockJobMasterContext); Assert.assertEquals(1, assignments.size()); Assert.assertEquals(10, assignments.values().iterator().next().size()); }
WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; for (int i = 0; i < workers.length - 1; i++) { workers[i] = new WorkerNetAddress().setHost(String.format("worker-%d", i)); workers[workers.length - 1] = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); when(mContext.acquireBlockWorkerClient(WORKER_NET_ADDRESS_LOCAL)) .thenThrow(new UnavailableException("failed to connect to " + WORKER_NET_ADDRESS_LOCAL.getHost())); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x ->
/** * Extracts dataPort socket address from Alluxio representation of network address. * * @param netAddress the input network address representation * @param conf Alluxio configuration * @return the socket address */ public static SocketAddress getDataPortSocketAddress(WorkerNetAddress netAddress, AlluxioConfiguration conf) { SocketAddress address; if (NettyUtils.isDomainSocketSupported(netAddress, conf)) { address = new DomainSocketAddress(netAddress.getDomainSocketPath()); } else { String host = netAddress.getHost(); int port = netAddress.getDataPort(); address = new InetSocketAddress(host, port); } return address; }
private BlockWorkerInfo worker(long capacity, String node, String rack) { WorkerNetAddress address = new WorkerNetAddress(); List<LocalityTier> tiers = new ArrayList<>(); if (node != null && !node.isEmpty()) { address.setHost(node); tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); } if (rack != null && !rack.isEmpty()) { tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); } address.setTieredIdentity(new TieredIdentity(tiers)); return new BlockWorkerInfo(address, capacity, 0); } }
/** * Constructs a new {@link TestBlockOutStream} to be used in tests. * * @param data the data to test * @param blockSize the block size */ public TestBlockOutStream(ByteBuffer data, long blockSize) { super(new TestDataWriter(data), blockSize, new WorkerNetAddress()); mData = data; mClosed = false; mCanceled = false; }
private static List<TargetInfo> getTargetInfos(String[] targets, AlluxioConfiguration conf) throws IOException { List<TargetInfo> targetInfoList = new ArrayList<>(); for (String target : targets) { if (target.equals(ROLE_MASTER)) { String masterHost = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_WEB, conf); int masterPort = NetworkAddressUtils.getPort(ServiceType.MASTER_WEB, conf); targetInfoList.add(new TargetInfo(masterHost, masterPort, ROLE_MASTER)); } else if (target.equals(ROLE_WORKERS)) { AlluxioBlockStore alluxioBlockStore = AlluxioBlockStore.create(FileSystemContext.create(ClientContext.create(conf))); List<BlockWorkerInfo> workerInfoList = alluxioBlockStore.getAllWorkers(); for (BlockWorkerInfo workerInfo : workerInfoList) { WorkerNetAddress netAddress = workerInfo.getNetAddress(); targetInfoList.add( new TargetInfo(netAddress.getHost(), netAddress.getWebPort(), ROLE_WORKER)); } } else if (target.contains(":")) { String[] hostPortPair = target.split(":"); int port = Integer.parseInt(hostPortPair[1]); targetInfoList.add(new TargetInfo(hostPortPair[0], port, ROLE_WORKER)); } else { throw new IOException("Unrecognized target argument: " + target); } } return targetInfoList; }
/** * @param workerNetAddress the worker address * @param conf Alluxio configuration * @return true if the domain socket is enabled on this client */ public static boolean isDomainSocketSupported(WorkerNetAddress workerNetAddress, AlluxioConfiguration conf) { if (workerNetAddress.getDomainSocketPath().isEmpty() || getUserChannel(conf) != ChannelType.EPOLL) { return false; } if (conf.getBoolean(PropertyKey.WORKER_DATA_SERVER_DOMAIN_SOCKET_AS_UUID)) { return FileUtils.exists(workerNetAddress.getDomainSocketPath()); } else { return workerNetAddress.getHost().equals(NetworkAddressUtils.getClientHostName(conf)); } }
@Test public void selectExecutorsTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10); WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); BlockLocation location = new BlockLocation(); location.setWorkerAddress(workerNetAddress); blockInfo.setLocations(Lists.newArrayList(location)); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); Map<WorkerInfo, SerializableVoid> result = new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(workerInfo), mMockJobMasterContext); Assert.assertEquals(1, result.size()); Assert.assertEquals(workerInfo, result.keySet().iterator().next()); }
@Test public void getBlockLocationsOnlyMatchingWorkers() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", worker2.getHost(), "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
/** * @param address the Alluxio worker network address * @param conf Alluxio configuration * @return true if the worker is local */ public static boolean isLocalHost(WorkerNetAddress address, AlluxioConfiguration conf) { return address.getHost().equals(NetworkAddressUtils.getClientHostName(conf)); }
@Test public void getBlockLocationsNoUfsLocationsDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }