private void init() { if (mInputStream != null) { return; } WorkerNetAddress address = new WorkerNetAddress().setHost(mDataSource.getHostName()) .setDataPort(mDataSource.getPort()); mInputStream = BlockInStream.createRemoteBlockInStream(mFsContext, mBlockId, address, BlockInStream.BlockInStreamSource.REMOTE, mBlockSize, mUfsOptions); mChannel = Channels.newChannel(mInputStream); } }
/** * Constructs a new {@link TestBlockOutStream} to be used in tests. * * @param data the data to test * @param blockSize the block size */ public TestBlockOutStream(ByteBuffer data, long blockSize) { super(new TestDataWriter(data), blockSize, new WorkerNetAddress()); mData = data; mClosed = false; mCanceled = false; }
public TestBlockInStream(byte[] data, long id, long length, boolean shortCircuit, BlockInStreamSource source) { super(new Factory(data, shortCircuit), new WorkerNetAddress(), source, id, length); mBytesRead = 0; }
/** * Constructs a new {@link TestUnderFileSystemFileOutStream} to be used in tests. * * @param data the data to test */ public TestUnderFileSystemFileOutStream(ByteBuffer data) { super(new TestDataWriter(data), new WorkerNetAddress()); mData = data; mClosed = false; mCanceled = false; }
/** * Sets up a new {@link MasterWorkerInfo} before a test runs. */ @Before public void before() { // register mInfo = new MasterWorkerInfo(0, new WorkerNetAddress()); mInfo.register(GLOBAL_STORAGE_TIER_ASSOC, STORAGE_TIER_ALIASES, TOTAL_BYTES_ON_TIERS, USED_BYTES_ON_TIERS, NEW_BLOCKS); }
@Test public void getBlockLocationsOnlyInAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoUfsLocationsDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsNoMatchingWorkersDefault() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Collections.EMPTY_LIST; verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsInUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsOnlyMatchingWorkers() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(); List<String> ufsLocations = Arrays.asList("worker0", worker2.getHost(), "worker3"); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker2); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Test public void getBlockLocationsInUfsAndAlluxio() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2").setDataPort(1234); List<WorkerNetAddress> blockWorkers = Arrays.asList(worker1); List<String> ufsLocations = Arrays.asList(worker2.getHost()); List<WorkerNetAddress> allWorkers = Arrays.asList(worker1, worker2); List<WorkerNetAddress> expectedWorkers = Arrays.asList(worker1); verifyBlockLocations(blockWorkers, ufsLocations, allWorkers, expectedWorkers); }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, ServerConfiguration.global())) .setRpcPort(mRpcAddress.getPort()) .setDataPort(getDataLocalPort()) .setDomainSocketPath(getDataDomainSocketPath()) .setWebPort(mWebServer.getLocalPort()) .setTieredIdentity(mTieredIdentitiy); }
@Test public void createDomainSocketEnabled() throws Exception { PowerMockito.mockStatic(NettyUtils.class); PowerMockito.when(NettyUtils.isDomainSocketSupported(Matchers.any(WorkerNetAddress.class), Matchers.any(InstancedConfiguration.class))) .thenReturn(true); WorkerNetAddress dataSource = new WorkerNetAddress(); BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.LOCAL; BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); Assert.assertFalse(stream.isShortCircuit()); } }
@Test public void createShortCircuit() throws Exception { WorkerNetAddress dataSource = new WorkerNetAddress(); BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.LOCAL; BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); Assert.assertTrue(stream.isShortCircuit()); }
@Test public void createRemote() throws Exception { WorkerNetAddress dataSource = new WorkerNetAddress(); BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.REMOTE; BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); Assert.assertFalse(stream.isShortCircuit()); }
@Override public WorkerNetAddress getAddress() { return new WorkerNetAddress() .setHost(NetworkAddressUtils.getConnectHost(ServiceType.JOB_WORKER_RPC, ServerConfiguration.global())) .setRpcPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_RPC_PORT)) .setDataPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_DATA_PORT)) .setWebPort(ServerConfiguration.getInt(PropertyKey.JOB_WORKER_WEB_PORT)); }
@Test public void createUfs() throws Exception { WorkerNetAddress dataSource = new WorkerNetAddress(); BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.UFS; BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); Assert.assertFalse(stream.isShortCircuit()); }
/** * Tests the {@link NetworkAddressUtils#getFqdnHost(InetSocketAddress)} and * {@link NetworkAddressUtils#getFqdnHost(WorkerNetAddress)} methods. */ @Test public void getFqdnHost() throws UnknownHostException { assertEquals(NetworkAddressUtils.getFqdnHost(new InetSocketAddress("localhost", 0)), "localhost"); assertEquals( NetworkAddressUtils.getFqdnHost(new WorkerNetAddress().setHost("localhost")), "localhost"); }
@Test public void createShortCircuitDisabled() throws Exception { try (Closeable c = new ConfigurationRule(PropertyKey.USER_SHORT_CIRCUIT_ENABLED, "false", mConf) .toResource()) { WorkerNetAddress dataSource = new WorkerNetAddress(); when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.LOCAL; BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions); Assert.assertFalse(stream.isShortCircuit()); } }
@Test public void getOutStreamRemote() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setLocationPolicy(new MockFileWriteLocationPolicy(Arrays.asList(worker1, worker2))) .setWriteType(WriteType.MUST_CACHE); BlockOutStream stream1 = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); assertEquals(worker1, stream1.getAddress()); BlockOutStream stream2 = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); assertEquals(worker2, stream2.getAddress()); }