FileInfo ret = new FileInfo(); ret.setFileId(getId()); ret.setName(getName()); ret.setPath(path); ret.setBlockSizeBytes(0); ret.setCreationTimeMs(getCreationTimeMs()); ret.setCompleted(true); ret.setFolder(isDirectory()); ret.setPinned(isPinned()); ret.setCacheable(false); ret.setPersisted(isPersisted()); ret.setLastModificationTimeMs(getLastModificationTimeMs()); ret.setTtl(mTtl); ret.setTtlAction(mTtlAction); ret.setOwner(getOwner()); ret.setGroup(getGroup()); ret.setMode(getMode()); ret.setPersistenceState(getPersistenceState().toString()); ret.setMountPoint(isMountPoint()); ret.setUfsFingerprint(Constants.INVALID_UFS_FINGERPRINT); ret.setAcl(mAcl); ret.setDefaultAcl(mDefaultAcl); return ret;
for (FileBlockInfo fileBlockInfo : fileInfo.getFileBlockInfos()) { fileBlockInfos.add(toProto(fileBlockInfo)); .setFileId(fileInfo.getFileId()).setName(fileInfo.getName()).setPath(fileInfo.getPath()) .setUfsPath(fileInfo.getUfsPath()).setLength(fileInfo.getLength()) .setBlockSizeBytes(fileInfo.getBlockSizeBytes()) .setCreationTimeMs(fileInfo.getCreationTimeMs()).setCompleted(fileInfo.isCompleted()) .setFolder(fileInfo.isFolder()).setPinned(fileInfo.isPinned()) .setCacheable(fileInfo.isCacheable()).setPersisted(fileInfo.isPersisted()) .addAllBlockIds(fileInfo.getBlockIds()) .setLastModificationTimeMs(fileInfo.getLastModificationTimeMs()).setTtl(fileInfo.getTtl()) .setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()).setMode(fileInfo.getMode()) .setPersistenceState(fileInfo.getPersistenceState()).setMountPoint(fileInfo.isMountPoint()) .addAllFileBlockInfos(fileBlockInfos) .setTtlAction(fileInfo.getTtlAction()).setMountId(fileInfo.getMountId()) .setInAlluxioPercentage(fileInfo.getInAlluxioPercentage()) .setInMemoryPercentage(fileInfo.getInMemoryPercentage()) .setUfsFingerprint(fileInfo.getUfsFingerprint()) .setReplicationMax(fileInfo.getReplicationMax()) .setReplicationMin(fileInfo.getReplicationMin()); if (!fileInfo.getAcl().equals(AccessControlList.EMPTY_ACL)) { builder.setAcl(toProto(fileInfo.getAcl())); if (!fileInfo.getDefaultAcl().equals(DefaultAccessControlList.EMPTY_DEFAULT_ACL)) { builder.setDefaultAcl(toProto(fileInfo.getDefaultAcl()));
AlluxioURI uri = inodePath.getUri(); FileInfo fileInfo = inode.generateClientFileInfo(uri.toString()); if (fileInfo.isFolder()) { fileInfo.setLength(inode.asDirectory().getChildCount()); fileInfo.setInMemoryPercentage(getInMemoryPercentage(inode)); fileInfo.setInAlluxioPercentage(getInAlluxioPercentage(inode)); if (inode.isFile()) { try { fileInfo.setFileBlockInfos(getFileBlockInfoListInternal(inodePath)); } catch (InvalidPathException e) { throw new FileDoesNotExistException(e.getMessage(), e); fileInfo.setUfsPath(resolvedUri.toString()); fileInfo.setMountId(resolution.getMountId()); Metrics.FILE_INFOS_GOT.inc(); return fileInfo;
/** * Creates a directory with the given name. * * @return file info for the created directory */ private FileInfo createDirectory(String name) throws Exception { // Call all directories mount points to force cross-mount functionality. FileInfo info = new FileInfo().setFolder(true).setPath(name).setMountPoint(true); when(mMockFileSystem.getStatus(new AlluxioURI(name))).thenReturn(new URIStatus(info)); return info; }
FileInfo fileInfo = FileInfoTest.createRandom(); URIStatus uriStatus = new URIStatus(fileInfo); Assert.assertEquals(uriStatus.getBlockIds(), fileInfo.getBlockIds()); Assert.assertEquals(uriStatus.getBlockSizeBytes(), fileInfo.getBlockSizeBytes()); Assert.assertEquals(uriStatus.getCreationTimeMs(), fileInfo.getCreationTimeMs()); Assert.assertEquals(uriStatus.getFileId(), fileInfo.getFileId()); Assert.assertEquals(uriStatus.getGroup(), fileInfo.getGroup()); Assert.assertEquals(uriStatus.getInMemoryPercentage(), fileInfo.getInMemoryPercentage()); Assert.assertEquals(uriStatus.getLastModificationTimeMs(), fileInfo.getLastModificationTimeMs()); Assert.assertEquals(uriStatus.getLength(), fileInfo.getLength()); Assert.assertEquals(uriStatus.getName(), fileInfo.getName()); Assert.assertEquals(uriStatus.getPath(), fileInfo.getPath()); Assert.assertEquals(uriStatus.getMode(), fileInfo.getMode()); Assert.assertEquals(uriStatus.getPersistenceState(), fileInfo.getPersistenceState()); Assert.assertEquals(uriStatus.getTtl(), fileInfo.getTtl()); Assert.assertEquals(uriStatus.getTtlAction(), fileInfo.getTtlAction()); Assert.assertEquals(uriStatus.getUfsPath(), fileInfo.getUfsPath()); Assert.assertEquals(uriStatus.getOwner(), fileInfo.getOwner()); Assert.assertEquals(uriStatus.isCacheable(), fileInfo.isCacheable()); Assert.assertEquals(uriStatus.isCompleted(), fileInfo.isCompleted()); Assert.assertEquals(uriStatus.isFolder(), fileInfo.isFolder()); Assert.assertEquals(uriStatus.isPersisted(), fileInfo.isPersisted()); Assert.assertEquals(uriStatus.isPinned(), fileInfo.isPinned()); Assert.assertEquals(uriStatus.isMountPoint(), fileInfo.isMountPoint());
/** * Tests the {@link MutableInodeDirectory#generateClientFileInfo(String)} method. */ @Test public void generateClientFileInfo() { MutableInodeDirectory inodeDirectory = createInodeDirectory(); String path = "/test/path"; FileInfo info = inodeDirectory.generateClientFileInfo(path); Assert.assertEquals(inodeDirectory.getId(), info.getFileId()); Assert.assertEquals(inodeDirectory.getName(), info.getName()); Assert.assertEquals(path, info.getPath()); Assert.assertEquals("", info.getUfsPath()); Assert.assertEquals(0, info.getLength()); Assert.assertEquals(0, info.getBlockSizeBytes()); Assert.assertEquals(inodeDirectory.getCreationTimeMs(), info.getCreationTimeMs()); Assert.assertTrue(info.isCompleted()); Assert.assertTrue(info.isFolder()); Assert.assertEquals(inodeDirectory.isPinned(), info.isPinned()); Assert.assertFalse(info.isCacheable()); Assert.assertNotNull(info.getBlockIds()); Assert.assertEquals(inodeDirectory.getLastModificationTimeMs(), info.getLastModificationTimeMs()); } }
@Test public void getStatus() throws Exception { FileInfo fileInfo = new FileInfo() .setLastModificationTimeMs(111L) .setFolder(false) .setOwner("user1") .setGroup("group1") .setMode(00755); Path path = new Path("/dir"); alluxio.client.file.FileSystem alluxioFs = mock(alluxio.client.file.FileSystem.class); when(alluxioFs.getStatus(new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)))) .thenReturn(new URIStatus(fileInfo)); FileSystem alluxioHadoopFs = new FileSystem(alluxioFs); FileStatus fileStatus = alluxioHadoopFs.getFileStatus(path); assertFileInfoEqualsFileStatus(fileInfo, fileStatus); }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
private void assertFileInfoEqualsFileStatus(FileInfo info, FileStatus status) { assertEquals(info.getOwner(), status.getOwner()); assertEquals(info.getGroup(), status.getGroup()); assertEquals(info.getMode(), status.getPermission().toShort()); assertEquals(info.getLastModificationTimeMs(), status.getModificationTime()); assertEquals(info.isFolder(), status.isDir()); } }
public void before() throws Exception { sConf = ConfigurationTestUtils.defaults(); mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(FILE_LENGTH); mInfo.setBlockIds(blockIds); mInfo.setFileBlockInfos(fileBlockInfos); mStatus = new URIStatus(mInfo);
private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception { FileInfo testFileInfo = new FileInfo(); AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int i = 0; i < numOfBlocks; i++) { blockInfos.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList()))); } testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); Mockito.when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; } }
@Test public void scheduleAsyncPersist() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(1, persistFiles.size()); assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds()); }
/** * Creates a mock {@link FileInStream} which will supply the given bytes. * * @param context file system context * @param bytes the bytes to supply */ public MockFileInStream(FileSystemContext context, byte[] bytes, AlluxioConfiguration conf) { super(new URIStatus(new FileInfo()), new InStreamOptions(new URIStatus(new FileInfo()), conf), context); mStream = new ByteArrayInputStream(bytes); }
String path = "/test"; URIStatus status = new URIStatus( new FileInfo().setPath(path).setBlockIds(Lists.newArrayList(TEST_BLOCK_ID)) .setFileBlockInfos(Lists.newArrayList( new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(TEST_BLOCK_ID))))); when(mMockFileSystem.getStatus(any(AlluxioURI.class))).thenReturn(status);
@Test public void getInStreamMissingBlock() throws Exception { URIStatus dummyStatus = new URIStatus( new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(NotFoundException.class); mException.expectMessage("unavailable in both Alluxio and UFS"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
mUfsManager.get(fileInfo.getMountId()).acquireUfsResource()) { UnderFileSystem ufs = ufsResource.get(); String dstPath = prepareUfsFilePath(fileInfo, ufs); OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults(ServerConfiguration.global()) .setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()) .setMode(new Mode((short) fileInfo.getMode()))); final WritableByteChannel outputChannel = Channels.newChannel(outputStream); List<Throwable> errors = new ArrayList<>();
/** * Tests that the worker will delete the source directory if the directory contains only * directories. */ @Test public void deleteDirsOnlySourceDir() throws Exception { String inner = TEST_DIR + "/innerDir"; when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))).thenReturn( Lists.newArrayList(new URIStatus(new FileInfo().setPath(inner).setFolder(true)))); when(mMockFileSystem.listStatus(new AlluxioURI(inner))) .thenReturn(Lists.<URIStatus>newArrayList()); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
@Test public void getInStreamRemote() throws Exception { WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1"); WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2"); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); // We should sometimes get remote1 and sometimes get remote2. Set<WorkerNetAddress> results = new HashSet<>(); for (int i = 0; i < 40; i++) { results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); } assertEquals(Sets.newHashSet(remote1, remote2), results); }
/** * For example, for the uri: alluxio://host:1000/foo/bar/baz, the path is /foo/bar/baz. * * @return the entire path component of the entity referenced by this uri, mutable */ public String getPath() { return mInfo.getPath(); }
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }