private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception { FileInfo testFileInfo = new FileInfo(); AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int i = 0; i < numOfBlocks; i++) { blockInfos.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList()))); } testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); Mockito.when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; } }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
/** * Creates a file with the given name and a block on each specified worker. Workers may be * repeated to give them multiple blocks. * * @param testFile the name of the file to create * @param fileInfo file info to apply to the created file * @param workerInds the workers to put blocks on, specified by their indices * @return file info for the created file */ private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo, int... workerInds) throws Exception { AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int workerInd : workerInds) { WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress(); blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo() .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address))))); } FileInfo testFileInfo = fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; }
URIStatus status = new URIStatus( new FileInfo().setPath(path).setBlockIds(Lists.newArrayList(TEST_BLOCK_ID)) .setFileBlockInfos(Lists.newArrayList( new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(TEST_BLOCK_ID))))); when(mMockFileSystem.getStatus(any(AlluxioURI.class))).thenReturn(status);
.setGroup("group1") .setMode(00755) .setFileBlockInfos(Arrays.asList(blockInfo)); Path path = new Path("/dir/file"); alluxio.client.file.FileSystem alluxioFs =
new URIStatus(new FileInfo().setPersisted(persisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }
result.setCompleted(completed); result.setCreationTimeMs(creationTimeMs); result.setFileBlockInfos(fileBlocksInfos); result.setFileId(fileId); result.setFolder(folder);
@Test public void selectExecutorsTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10); WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); BlockLocation location = new BlockLocation(); location.setWorkerAddress(workerNetAddress); blockInfo.setLocations(Lists.newArrayList(location)); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); Map<WorkerInfo, SerializableVoid> result = new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(workerInfo), mMockJobMasterContext); Assert.assertEquals(1, result.size()); Assert.assertEquals(workerInfo, result.keySet().iterator().next()); }
mInfo.setFileBlockInfos(fileBlockInfos); mStatus = new URIStatus(mInfo);
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
if (inode.isFile()) { try { fileInfo.setFileBlockInfos(getFileBlockInfoListInternal(inodePath)); } catch (InvalidPathException e) { throw new FileDoesNotExistException(e.getMessage(), e);
/** * Converts a proto type to a wire type. * * @param pInfo the proto representation of a file information * @return wire representation of the file information */ public static FileInfo fromProto(alluxio.grpc.FileInfo pInfo) { return new FileInfo().setFileId(pInfo.getFileId()).setName(pInfo.getName()) .setPath(pInfo.getPath()).setUfsPath(pInfo.getUfsPath()).setLength(pInfo.getLength()) .setBlockSizeBytes(pInfo.getBlockSizeBytes()).setCreationTimeMs(pInfo.getCreationTimeMs()) .setCompleted(pInfo.getCompleted()).setFolder(pInfo.getFolder()) .setPinned(pInfo.getPinned()).setCacheable(pInfo.getCacheable()) .setPersisted(pInfo.getPersisted()).setBlockIds(pInfo.getBlockIdsList()) .setLastModificationTimeMs(pInfo.getLastModificationTimeMs()).setTtl(pInfo.getTtl()) .setTtlAction(pInfo.getTtlAction()).setOwner(pInfo.getOwner()) .setGroup(pInfo.getGroup()).setMode(pInfo.getMode()) .setPersistenceState(pInfo.getPersistenceState()).setMountPoint(pInfo.getMountPoint()) .setFileBlockInfos(map(GrpcUtils::fromProto, pInfo.getFileBlockInfosList())) .setMountId(pInfo.getMountId()).setInAlluxioPercentage(pInfo.getInAlluxioPercentage()) .setInMemoryPercentage(pInfo.getInMemoryPercentage()) .setUfsFingerprint(pInfo.hasUfsFingerprint() ? pInfo.getUfsFingerprint() : Constants.INVALID_UFS_FINGERPRINT) .setAcl(pInfo.hasAcl() ? (fromProto(pInfo.getAcl())) : AccessControlList.EMPTY_ACL) .setDefaultAcl( pInfo.hasDefaultAcl() ? ((DefaultAccessControlList) fromProto(pInfo.getDefaultAcl())) : DefaultAccessControlList.EMPTY_DEFAULT_ACL) .setReplicationMax(pInfo.getReplicationMax()).setReplicationMin(pInfo.getReplicationMin()); }
if (inode instanceof InodeFile) { try { fileInfo.setFileBlockInfos(getFileBlockInfoListInternal(inodePath)); } catch (InvalidPathException e) { throw new FileDoesNotExistException(e.getMessage(), e);
.setPersistenceState(info.getPersistenceState()) .setMountPoint(info.isMountPoint()) .setFileBlockInfos(map(FileBlockInfo::fromThrift, info.getFileBlockInfos())) .setMountId(info.getMountId()) .setInAlluxioPercentage(info.getInAlluxioPercentage())