/** * @param blockWorkers a list of block workers * @param blockInfo information about a block * @return the block worker hosts which are not storing the specified block */ private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers, FileBlockInfo blockInfo) { List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream() .map(location -> location.getWorkerAddress().getHost()) .collect(Collectors.toList()); return blockWorkers.stream() .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost())) .map(worker -> worker.getNetAddress().getHost()) .collect(Collectors.toList()); }
@Test public void selectExecutorsAllWorkers() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper(Lists .newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2), new BlockLocation().setWorkerAddress(ADDRESS_3)), 3, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_1, null); expected.put(WORKER_INFO_2, null); expected.put(WORKER_INFO_3, null); // Expect all workers are selected as they all have this block Assert.assertEquals(expected, result); }
/** * Converts wire type to proto type. * * @param blockLocation the wire type to convert * @return the converted proto type */ public static alluxio.grpc.BlockLocation toProto(BlockLocation blockLocation) { return alluxio.grpc.BlockLocation.newBuilder().setWorkerId(blockLocation.getWorkerId()) .setWorkerAddress(toProto(blockLocation.getWorkerAddress())) .setTierAlias(blockLocation.getTierAlias()).build(); }
/** * Converts a proto type to a wire type. * * @param blockPLocation the proto type to convert * @return the converted wire type */ public static BlockLocation fromProto(alluxio.grpc.BlockLocation blockPLocation) { BlockLocation blockLocation = new BlockLocation(); blockLocation.setWorkerId(blockPLocation.getWorkerId()); blockLocation.setWorkerAddress(fromProto(blockPLocation.getWorkerAddress())); blockLocation.setTierAlias(blockPLocation.getTierAlias()); return blockLocation; }
/** * Tests the persistence of file with block on multiple workers. */ @Test public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location1 = new BlockLocation().setWorkerId(1); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1)))); BlockLocation location2 = new BlockLocation().setWorkerId(2); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2)))); long fileId = 2; when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); // no persist scheduled on any worker assertEquals(0, handler.pollFilesToPersist(1).size()); assertEquals(0, handler.pollFilesToPersist(2).size()); }
/** * @return true if the given block is in the top storage level in some worker, false otherwise */ private boolean isInTopStorageTier(BlockInfo blockInfo) { for (BlockLocation location : blockInfo.getLocations()) { if (mBlockMaster.getGlobalStorageTierAssoc().getOrdinal(location.getTierAlias()) == 0) { return true; } } return false; }
if (workerBlockCounts.containsKey(blockLocation.getWorkerId())) { workerBlockCounts.put(blockLocation.getWorkerId(), workerBlockCounts.get(blockLocation.getWorkerId()) + 1); } else { workerBlockCounts.put(blockLocation.getWorkerId(), 1); if (workerBlockCounts.get(blockLocation.getWorkerId()) == blockInfoList.size()) { return blockLocation.getWorkerId();
public static BlockLocation createRandom() { BlockLocation result = new BlockLocation(); Random random = new Random(); long workerId = random.nextLong(); WorkerNetAddress workerAddress = WorkerNetAddressTest.createRandom(); String tierAlias = CommonUtils.randomAlphaNumString(random.nextInt(10)); result.setWorkerId(workerId); result.setWorkerAddress(workerAddress); result.setTierAlias(tierAlias); return result; } }
@Test public void scheduleAsyncPersist() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(1, persistFiles.size()); assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds()); }
/** * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
if (workerBlockCounts.containsKey(blockLocation.getWorkerId())) { workerBlockCounts.put(blockLocation.getWorkerId(), workerBlockCounts.get(blockLocation.getWorkerId()) + 1); } else { workerBlockCounts.put(blockLocation.getWorkerId(), 1); if (workerBlockCounts.get(blockLocation.getWorkerId()) == blockInfoList.size()) { return blockLocation.getWorkerId();
@Test public void selectExecutorsAnyOneWorkers() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper(Lists .newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2), new BlockLocation().setWorkerAddress(ADDRESS_3)), 1, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); // Expect one worker from all workers having this block Assert.assertEquals(1, result.size()); Assert.assertEquals(null, result.values().iterator().next()); }
locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()) .setWorkerAddress(workerInfo.getWorkerAddress()) .setTierAlias(location.getTier()));
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
public void checkEquality(BlockLocation a, BlockLocation b) { Assert.assertEquals(a.getWorkerId(), b.getWorkerId()); Assert.assertEquals(a.getWorkerAddress(), b.getWorkerAddress()); Assert.assertEquals(a.getTierAlias(), b.getTierAlias()); Assert.assertEquals(a, b); }
/** * Tests persistence after deletion of files. */ @Test public void persistenceFileAfterDeletion() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenThrow(new FileDoesNotExistException("no file")); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(0, persistFiles.size()); } }
/** * @return true if the given block is in the top storage level in some worker, false otherwise */ private boolean isInTopStorageTier(BlockInfo blockInfo) { for (BlockLocation location : blockInfo.getLocations()) { if (mBlockMaster.getGlobalStorageTierAssoc().getOrdinal(location.getTierAlias()) == 0) { return true; } } return false; }
@Test public void selectExecutorsBothWorkers() throws Exception { Map<WorkerInfo, SerializableVoid> result = selectExecutorsTestHelper(Lists .newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2)), 3, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3)); Map<WorkerInfo, SerializableVoid> expected = Maps.newHashMap(); expected.put(WORKER_INFO_1, null); expected.put(WORKER_INFO_2, null); // Expect both workers having this block should be selected Assert.assertEquals(expected, result); } }
@Test public void getBlockInfo() throws Exception { // Create a worker with a block. long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); long blockId = 1L; long blockLength = 20L; mBlockMaster.workerRegister(worker1, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(worker1, 50L, "MEM", blockId, blockLength); BlockLocation blockLocation = new BlockLocation() .setTierAlias("MEM") .setWorkerAddress(NET_ADDRESS_1) .setWorkerId(worker1); BlockInfo expectedBlockInfo = new BlockInfo() .setBlockId(1L) .setLength(20L) .setLocations(ImmutableList.of(blockLocation)); assertEquals(expectedBlockInfo, mBlockMaster.getBlockInfo(blockId)); }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); Preconditions.checkArgument(numReplicas > 0); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that don't have this block locally to replicate if (!hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }