public static BlockLocation createRandom() { BlockLocation result = new BlockLocation(); Random random = new Random(); long workerId = random.nextLong(); WorkerNetAddress workerAddress = WorkerNetAddressTest.createRandom(); String tierAlias = CommonUtils.randomAlphaNumString(random.nextInt(10)); result.setWorkerId(workerId); result.setWorkerAddress(workerAddress); result.setTierAlias(tierAlias); return result; } }
/** * Converts a proto type to a wire type. * * @param blockPLocation the proto type to convert * @return the converted wire type */ public static BlockLocation fromProto(alluxio.grpc.BlockLocation blockPLocation) { BlockLocation blockLocation = new BlockLocation(); blockLocation.setWorkerId(blockPLocation.getWorkerId()); blockLocation.setWorkerAddress(fromProto(blockPLocation.getWorkerAddress())); blockLocation.setTierAlias(blockPLocation.getTierAlias()); return blockLocation; }
locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()) .setWorkerAddress(workerInfo.getWorkerAddress()) .setTierAlias(location.getTier()));
@Test public void scheduleAsyncPersist() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(1, persistFiles.size()); assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds()); }
/** * Tests the persistence of file with block on multiple workers. */ @Test public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location1 = new BlockLocation().setWorkerId(1); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1)))); BlockLocation location2 = new BlockLocation().setWorkerId(2); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2)))); long fileId = 2; when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); // no persist scheduled on any worker assertEquals(0, handler.pollFilesToPersist(1).size()); assertEquals(0, handler.pollFilesToPersist(2).size()); }
/** * Tests persistence after deletion of files. */ @Test public void persistenceFileAfterDeletion() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenThrow(new FileDoesNotExistException("no file")); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(0, persistFiles.size()); } }
@Test public void getBlockInfo() throws Exception { // Create a worker with a block. long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); long blockId = 1L; long blockLength = 20L; mBlockMaster.workerRegister(worker1, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(worker1, 50L, "MEM", blockId, blockLength); BlockLocation blockLocation = new BlockLocation() .setTierAlias("MEM") .setWorkerAddress(NET_ADDRESS_1) .setWorkerId(worker1); BlockInfo expectedBlockInfo = new BlockInfo() .setBlockId(1L) .setLength(20L) .setLocations(ImmutableList.of(blockLocation)); assertEquals(expectedBlockInfo, mBlockMaster.getBlockInfo(blockId)); }
/** * Creates a new instance of {@link BlockLocation} from a thrift representation. * * @param blockLocation the thrift representation of a block location * @return the instance */ public static BlockLocation fromThrift(alluxio.thrift.BlockLocation blockLocation) { return new BlockLocation() .setWorkerId(blockLocation.getWorkerId()) .setWorkerAddress(WorkerNetAddress.fromThrift(blockLocation.getWorkerAddress())) .setTierAlias(blockLocation.getTierAlias()); }
locations.add(new BlockLocation().setWorkerId(masterBlockLocation.getWorkerId()) .setWorkerAddress(workerInfo.getWorkerAddress()) .setTierAlias(masterBlockLocation.getTierAlias()));