@Test public void selectExecutorsAllWorkers() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper(Lists .newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2), new BlockLocation().setWorkerAddress(ADDRESS_3)), 3, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_1, null); expected.put(WORKER_INFO_2, null); expected.put(WORKER_INFO_3, null); // Expect all workers are selected as they all have this block Assert.assertEquals(expected, result); }
@Test public void selectExecutorsAnyOneWorkers() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper(Lists .newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2), new BlockLocation().setWorkerAddress(ADDRESS_3)), 1, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); // Expect one worker from all workers having this block Assert.assertEquals(1, result.size()); Assert.assertEquals(null, result.values().iterator().next()); }
@Test public void selectExecutorsBothWorkers() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper(Lists .newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2)), 3, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_1, null); expected.put(WORKER_INFO_2, null); // Expect both workers having this block should be selected Assert.assertEquals(expected, result); } }
@Test public void selectExecutorsTwoWorkersValid() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 2, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_2, null); expected.put(WORKER_INFO_3, null); // select both workers left assertEquals(expected, result); }
@Test public void selectExecutorsInsufficientWorkerValid() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 2, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_2, null); // select the only worker left though more copies are requested assertEquals(expected, result); }
@Test public void selectExecutorsOnlyOneBlockWorkerHasBlock() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 1, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_1, null); // Expect the only worker 1 having this block Assert.assertEquals(expected, result); }
@Test public void selectExecutorsOnlyOneWorkerValid() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 1, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_2, null); // select one worker left assertEquals(expected, result); }
@Test public void selectExecutorsOneOutOFTwoWorkersValid() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 1, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); // select one worker out of two assertEquals(1, result.size()); assertEquals(null, result.values().iterator().next()); }
@Test public void selectExecutorsNoWorkerValid() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 1, Lists.newArrayList(WORKER_INFO_1)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); // select none as no choice left assertEquals(expected, result); }
public static BlockLocation createRandom() { BlockLocation result = new BlockLocation(); Random random = new Random(); long workerId = random.nextLong(); WorkerNetAddress workerAddress = WorkerNetAddressTest.createRandom(); String tierAlias = CommonUtils.randomAlphaNumString(random.nextInt(10)); result.setWorkerId(workerId); result.setWorkerAddress(workerAddress); result.setTierAlias(tierAlias); return result; } }
@Test public void selectExecutorsNoJobWorkerHasBlock() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper( Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 1, Lists.newArrayList(WORKER_INFO_2, WORKER_INFO_3)); // Expect none as no worker that is available has this copy Assert.assertEquals(EMPTY, result); }
/** * Converts a proto type to a wire type. * * @param blockPLocation the proto type to convert * @return the converted wire type */ public static BlockLocation fromProto(alluxio.grpc.BlockLocation blockPLocation) { BlockLocation blockLocation = new BlockLocation(); blockLocation.setWorkerId(blockPLocation.getWorkerId()); blockLocation.setWorkerAddress(fromProto(blockPLocation.getWorkerAddress())); blockLocation.setTierAlias(blockPLocation.getTierAlias()); return blockLocation; }
/** * Creates a file with the given name and a block on each specified worker. Workers may be * repeated to give them multiple blocks. * * @param testFile the name of the file to create * @param fileInfo file info to apply to the created file * @param workerInds the workers to put blocks on, specified by their indices * @return file info for the created file */ private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo, int... workerInds) throws Exception { AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int workerInd : workerInds) { WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress(); blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo() .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address))))); } FileInfo testFileInfo = fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; }
/** * Tests persistence after deletion of files. */ @Test public void persistenceFileAfterDeletion() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenThrow(new FileDoesNotExistException("no file")); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(0, persistFiles.size()); } }
@Test public void scheduleAsyncPersist() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(1, persistFiles.size()); assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds()); }
@Test public void getInStreamRemote() throws Exception { WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1"); WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2"); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); // We should sometimes get remote1 and sometimes get remote2. Set<WorkerNetAddress> results = new HashSet<>(); for (int i = 0; i < 40; i++) { results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); } assertEquals(Sets.newHashSet(remote1, remote2), results); }
/** * Tests the persistence of file with block on multiple workers. */ @Test public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location1 = new BlockLocation().setWorkerId(1); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1)))); BlockLocation location2 = new BlockLocation().setWorkerId(2); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2)))); long fileId = 2; when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); // no persist scheduled on any worker assertEquals(0, handler.pollFilesToPersist(1).size()); assertEquals(0, handler.pollFilesToPersist(2).size()); }
@Test public void selectExecutorsTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10); WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); BlockLocation location = new BlockLocation(); location.setWorkerAddress(workerNetAddress); blockInfo.setLocations(Lists.newArrayList(location)); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); Map<WorkerInfo, SerializableVoid> result = new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(workerInfo), mMockJobMasterContext); Assert.assertEquals(1, result.size()); Assert.assertEquals(workerInfo, result.keySet().iterator().next()); }
@Test public void getInStreamLocal() throws Exception { WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); // Mock away gRPC usage. OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build(); when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(response); observer.onCompleted(); return mStreamObserver; } }); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); }
@Test public void getBlockInfo() throws Exception { // Create a worker with a block. long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); long blockId = 1L; long blockLength = 20L; mBlockMaster.workerRegister(worker1, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(worker1, 50L, "MEM", blockId, blockLength); BlockLocation blockLocation = new BlockLocation() .setTierAlias("MEM") .setWorkerAddress(NET_ADDRESS_1) .setWorkerId(worker1); BlockInfo expectedBlockInfo = new BlockInfo() .setBlockId(1L) .setLength(20L) .setLocations(ImmutableList.of(blockLocation)); assertEquals(expectedBlockInfo, mBlockMaster.getBlockInfo(blockId)); }