final INodeDirectory rootDir = sourceNamesystem.dir.rootDir; final long numINodes = rootDir.getDirectoryWithQuotaFeature() .getSpaceConsumed().getNameSpace(); String sdPath = newFile.getParentFile().getParentFile().getAbsolutePath(); Step step = new Step(StepType.INODES, sdPath);
private static QuotaUsage getQuotaUsageInt(FSDirectory fsd, INodesInPath iip) throws IOException { fsd.readLock(); try { INode targetNode = iip.getLastINode(); QuotaUsage usage = null; if (targetNode.isDirectory()) { DirectoryWithQuotaFeature feature = targetNode.asDirectory().getDirectoryWithQuotaFeature(); if (feature != null) { QuotaCounts counts = feature.getSpaceConsumed(); QuotaCounts quotas = feature.getQuota(); usage = new QuotaUsage.Builder(). fileAndDirectoryCount(counts.getNameSpace()). quota(quotas.getNameSpace()). spaceConsumed(counts.getStorageSpace()). spaceQuota(quotas.getStorageSpace()). typeConsumed(counts.getTypeSpaces().asArray()). typeQuota(quotas.getTypeSpaces().asArray()).build(); } } return usage; } finally { fsd.readUnlock(); } }
long totalInodes() { readLock(); try { return rootDir.getDirectoryWithQuotaFeature().getSpaceConsumed() .getNameSpace(); } finally { readUnlock(); } }
long totalInodes() { readLock(); try { return rootDir.getDirectoryWithQuotaFeature().getSpaceConsumed() .getNameSpace(); } finally { readUnlock(); } }
private void scanDirsWithQuota(INodeDirectory dir, HashMap<String, Long> nsMap, HashMap<String, Long> dsMap, boolean verify) { if (dir.isQuotaSet()) { // get the current consumption QuotaCounts q = dir.getDirectoryWithQuotaFeature().getSpaceConsumed(); String name = dir.getFullPathName(); if (verify) { assertEquals(nsMap.get(name).longValue(), q.getNameSpace()); assertEquals(dsMap.get(name).longValue(), q.getStorageSpace()); } else { nsMap.put(name, Long.valueOf(q.getNameSpace())); dsMap.put(name, Long.valueOf(q.getStorageSpace())); } } for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) { if (child instanceof INodeDirectory) { scanDirsWithQuota((INodeDirectory)child, nsMap, dsMap, verify); } } }
@Test(timeout = 60000) public void testQuotaByStorageTypeParentOffChildOff() throws Exception { final Path parent = new Path(dir, "parent"); final Path child = new Path(parent, "child"); dfs.mkdirs(parent); dfs.mkdirs(child); dfs.setStoragePolicy(parent, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); // Create file of size 2.5 * BLOCKSIZE under child directory. // Since both parent and child directory do not have SSD quota set, // expect succeed without exception Path createdFile1 = new Path(child, "created_file1.data"); long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2; int bufLen = BLOCKSIZE / 16; DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed); // Verify SSD usage at the root level as both parent/child don't have DirectoryWithQuotaFeature INode fnode = fsdir.getINode4Write("/"); long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumed); }
private void testTruncate(long newLength, long expectedDiff, long expectedUsage) throws Exception { // before doing the real truncation, make sure the computation is correct final INodesInPath iip = fsdir.getINodesInPath4Write(file.toString()); final INodeFile fileNode = iip.getLastINode().asFile(); fileNode.recordModification(iip.getLatestSnapshotId(), true); final long diff = fileNode.computeQuotaDeltaForTruncate(newLength); Assert.assertEquals(expectedDiff, diff); // do the real truncation dfs.truncate(file, newLength); // wait for truncate to finish TestFileTruncate.checkBlockRecovery(file, dfs); final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString()) .asDirectory(); final long spaceUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getStorageSpace(); final long diskUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(StorageType.DISK); Assert.assertEquals(expectedUsage, spaceUsed); Assert.assertEquals(expectedUsage, diskUsed); }
void testQuotaByStorageTypeWithFileCreateCase( String storagePolicy, StorageType storageType, short replication) throws Exception { final Path foo = new Path(dir, "foo"); Path createdFile1 = new Path(foo, "created_file1.data"); dfs.mkdirs(foo); // set storage policy on directory "foo" to storagePolicy dfs.setStoragePolicy(foo, storagePolicy); // set quota by storage type on directory "foo" dfs.setQuotaByStorageType(foo, storageType, BLOCKSIZE * 10); INode fnode = fsdir.getINode4Write(foo.toString()); assertTrue(fnode.isDirectory()); assertTrue(fnode.isQuotaSet()); // Create file of size 2 * BLOCKSIZE under directory "foo" long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2; int bufLen = BLOCKSIZE / 16; DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed); // Verify space consumed and remaining quota long storageTypeConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(storageType); assertEquals(file1Len * replication, storageTypeConsumed); }
private void testQuotaByStorageTypeOrTraditionalQuotaExceededCase( long storageSpaceQuotaInBlocks, long ssdQuotaInBlocks, long testFileLenInBlocks, short replication) throws Exception { final String METHOD_NAME = GenericTestUtils.getMethodName(); final Path testDir = new Path(dir, METHOD_NAME); dfs.mkdirs(testDir); dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); final long ssdQuota = BLOCKSIZE * ssdQuotaInBlocks; final long storageSpaceQuota = BLOCKSIZE * storageSpaceQuotaInBlocks; dfs.setQuota(testDir, Long.MAX_VALUE - 1, storageSpaceQuota); dfs.setQuotaByStorageType(testDir, StorageType.SSD, ssdQuota); INode testDirNode = fsdir.getINode4Write(testDir.toString()); assertTrue(testDirNode.isDirectory()); assertTrue(testDirNode.isQuotaSet()); Path createdFile = new Path(testDir, "created_file.data"); long fileLen = testFileLenInBlocks * BLOCKSIZE; try { DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, fileLen, BLOCKSIZE, replication, seed); fail("Should have failed with DSQuotaExceededException or " + "QuotaByStorageTypeExceededException "); } catch (Throwable t) { LOG.info("Got expected exception ", t); long currentSSDConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(Math.min(ssdQuota, storageSpaceQuota/replication), currentSSDConsumed); } }
assertTrue(fnode.isQuotaSet()); long currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, currentSSDConsumed); LOG.info("Got expected exception ", t); currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, currentSSDConsumed);
private void checkQuotaUsageComputation(final Path dirPath, final long expectedNs, final long expectedDs) throws IOException { INodeDirectory dirNode = getDir(fsdir, dirPath); assertTrue(dirNode.isQuotaSet()); QuotaCounts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs, q.getNameSpace()); assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs, q.getStorageSpace()); QuotaCounts counts = new QuotaCounts.Builder().build(); dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false); assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs, counts.getNameSpace()); assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs, counts.getStorageSpace()); }
/** * Test if the quota can be correctly updated for create file */ @Test (timeout=60000) public void testQuotaUpdateWithFileCreate() throws Exception { final Path foo = new Path(getParent(GenericTestUtils.getMethodName()), "foo"); Path createdFile = new Path(foo, "created_file.data"); getDFS().mkdirs(foo); getDFS().setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2; DFSTestUtil.createFile(getDFS(), createdFile, BLOCKSIZE / 16, fileLen, BLOCKSIZE, REPLICATION, seed); INode fnode = getFSDirectory().getINode4Write(foo.toString()); assertTrue(fnode.isDirectory()); assertTrue(fnode.isQuotaSet()); QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature() .getSpaceConsumed(); assertEquals(2, cnt.getNameSpace()); assertEquals(fileLen * REPLICATION, cnt.getStorageSpace()); }
.getSpaceConsumed(); assertEquals(2, cnt.getNameSpace()); assertEquals(fileLen * REPLICATION, cnt.getStorageSpace()); .getSpaceConsumed(); assertEquals(1, cntAfterDelete.getNameSpace()); assertEquals(0, cntAfterDelete.getStorageSpace());
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumed); .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(newFile1Len, ssdConsumed);
getFSDirectory().getINode4Write(dir.toString()).asDirectory(); final long spaceUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getStorageSpace(); try { getDFS().truncate(file, BLOCKSIZE / 2 - 1); .getSpaceConsumed().getStorageSpace(); assertEquals(spaceUsed, newSpaceUsed);
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumed); .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumed);
getFSDirectory().getINode4Write(dir.toString()).asDirectory(); final long spaceUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getStorageSpace(); try { DFSTestUtil.appendFile(getDFS(), file, BLOCKSIZE); .getSpaceConsumed().getStorageSpace(); assertEquals(spaceUsed, newSpaceUsed);
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumed); .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumedAfterNNRestart);
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, storageTypeConsumed); .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(0, storageTypeConsumed);
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumed); .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD); assertEquals(file1Len, ssdConsumedAfterNNRestart);