private long getSpace(FileStatus f) { Long cached = directorySpaces.get(f); if (cached != null) { return cached; } try { long space = f.isDirectory() ? fs.getContentSummary(f.getPath()).getSpaceConsumed() : f.getLen(); directorySpaces.put(f, space); return space; } catch (IOException e) { LOG.trace("Failed to get space consumed by path={}", f, e); return -1; } } });
@Override @InterfaceAudience.Private public void write(DataOutput out) throws IOException { out.writeLong(length); out.writeLong(fileCount); out.writeLong(directoryCount); out.writeLong(getQuota()); out.writeLong(getSpaceConsumed()); out.writeLong(getSpaceQuota()); }
/** {@inheritDoc} */ @Override public long usedSpaceSize() { try { // We don't use FileSystem#getUsed() since it counts only the files // in the filesystem root, not all the files recursively. return fileSystemForUser().getContentSummary(new Path("/")).getSpaceConsumed(); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to get used space size of file system."); } }
@Override protected void processPath(PathData item) throws IOException { ContentSummary contentSummary = item.fs.getContentSummary(item.path); long length = contentSummary.getLength(); long spaceConsumed = contentSummary.getSpaceConsumed(); if (excludeSnapshots) { length -= contentSummary.getSnapshotLength(); spaceConsumed -= contentSummary.getSnapshotSpaceConsumed(); } getUsagesTable().addRow(formatSize(length), formatSize(spaceConsumed), item); } }
srcSummary.getSpaceConsumed(), dstSummary.getSpaceConsumed());
/** Convert a ContentSummary to a Json string. */ public static String toJsonString(final ContentSummary contentsummary) { if (contentsummary == null) { return null; } final Map<String, Object> m = new TreeMap<String, Object>(); m.put("length", contentsummary.getLength()); m.put("fileCount", contentsummary.getFileCount()); m.put("directoryCount", contentsummary.getDirectoryCount()); m.put("quota", contentsummary.getQuota()); m.put("spaceConsumed", contentsummary.getSpaceConsumed()); m.put("spaceQuota", contentsummary.getSpaceQuota()); final Map<String, Map<String, Long>> typeQuota = new TreeMap<String, Map<String, Long>>(); for (StorageType t : StorageType.getTypesSupportingQuota()) { long tQuota = contentsummary.getTypeQuota(t); if (tQuota != HdfsConstants.QUOTA_RESET) { Map<String, Long> type = typeQuota.get(t.toString()); if (type == null) { type = new TreeMap<String, Long>(); typeQuota.put(t.toString(), type); } type.put("quota", contentsummary.getTypeQuota(t)); type.put("consumed", contentsummary.getTypeConsumed(t)); } } m.put("typeQuota", typeQuota); return toJsonString(ContentSummary.class, m); }
/** * * @param path * @return number of bytes stored in this subtree in bytes * @throws IOException */ public long getUsedQuotaInMbs(Path path) throws IOException { return dfs.getContentSummary(path).getSpaceConsumed() / DistributedFileSystemOps.MB; }
public void setInputPathSize(String inputPath) throws IOException { long fileSize = 0; for (FileStatus fileStatus : this.fsArray) { if (fileStatus.getPath().getName().startsWith("_") || fileStatus.getPath().getName().startsWith(".")) { continue; } fileSize = fileSize + this.fs.getContentSummary(fileStatus.getPath()).getSpaceConsumed(); } this.inputPathSize = (long) (fileSize * this.inputCompressionRatio); }
public void setInputPathSize(String inputPath, String inputCompression, String inputSerialization) throws IOException { long fileSize = 0; this.setInputCompressionRatio(inputCompression, inputSerialization); for (FileStatus fileStatus : this.fsArray) { if (fileStatus.getPath().getName().startsWith("_") || fileStatus.getPath().getName().startsWith(".")) { continue; } fileSize = fileSize + this.fs.getContentSummary(fileStatus.getPath()).getSpaceConsumed(); } this.inputPathSize = (long) (fileSize * this.inputCompressionRatio); }
@Override protected void postExecution(boolean succeed) throws IOException, StorageEngineException { super.postExecution(succeed); if (succeed) { if (input.equalsIgnoreCase("archive") && StringUtils.isNotEmpty(outputPath)) { FileSystem fs = FileSystem.get(getConf()); ContentSummary contentSummary = fs.getContentSummary(new Path(outputPath)); logger.info("Generated file " + outputPath); logger.info(" - Size (HDFS) : " + humanReadableByteCount(contentSummary.getLength(), false)); logger.info(" - SpaceConsumed (raw) : " + humanReadableByteCount(contentSummary.getSpaceConsumed(), false)); } } }
/** Convert a ContentSummary to a Json string. */ public static String toJsonString(final ContentSummary contentsummary) { if (contentsummary == null) { return null; } final Map<String, Object> m = new TreeMap<String, Object>(); m.put("length", contentsummary.getLength()); m.put("fileCount", contentsummary.getFileCount()); m.put("directoryCount", contentsummary.getDirectoryCount()); m.put("quota", contentsummary.getQuota()); m.put("spaceConsumed", contentsummary.getSpaceConsumed()); m.put("spaceQuota", contentsummary.getSpaceQuota()); return toJsonString(ContentSummary.class, m); }
/** Convert a ContentSummary to a Json string. */ public static String toJsonString(final ContentSummary contentsummary) { if (contentsummary == null) { return null; } final Map<String, Object> m = new TreeMap<String, Object>(); m.put("length", contentsummary.getLength()); m.put("fileCount", contentsummary.getFileCount()); m.put("directoryCount", contentsummary.getDirectoryCount()); m.put("quota", contentsummary.getQuota()); m.put("spaceConsumed", contentsummary.getSpaceConsumed()); m.put("spaceQuota", contentsummary.getSpaceQuota()); return toJsonString(ContentSummary.class, m); }
@Test(timeout = 60000) public void testContentSummaryWithoutQuotaByStorageType() throws Exception { final Path foo = new Path(dir, "foo"); Path createdFile1 = new Path(foo, "created_file1.data"); dfs.mkdirs(foo); // set storage policy on directory "foo" to ONESSD dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); INode fnode = fsdir.getINode4Write(foo.toString()); assertTrue(fnode.isDirectory()); assertTrue(!fnode.isQuotaSet()); // Create file of size 2 * BLOCKSIZE under directory "foo" long file1Len = BLOCKSIZE * 2; int bufLen = BLOCKSIZE / 16; DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed); // Verify getContentSummary without any quota set ContentSummary cs = dfs.getContentSummary(foo); assertEquals(cs.getSpaceConsumed(), file1Len * REPLICATION); assertEquals(cs.getTypeConsumed(StorageType.SSD), file1Len); assertEquals(cs.getTypeConsumed(StorageType.DISK), file1Len * 2); }
@Test public void testTruncate() throws Exception { final short repl = 3; final int blockSize = 1024; final int numOfBlocks = 2; DistributedFileSystem fs = cluster.getFileSystem(); Path dir = getTestRootPath(fc, "test/hadoop"); Path file = getTestRootPath(fc, "test/hadoop/file"); final byte[] data = FileSystemTestHelper.getFileData( numOfBlocks, blockSize); FileSystemTestHelper.createFile(fs, file, data, blockSize, repl); final int newLength = blockSize; boolean isReady = fc.truncate(file, newLength); Assert.assertTrue("Recovery is not expected.", isReady); FileStatus fileStatus = fc.getFileStatus(file); Assert.assertEquals(fileStatus.getLen(), newLength); AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString()); ContentSummary cs = fs.getContentSummary(dir); Assert.assertEquals("Bad disk space usage", cs.getSpaceConsumed(), newLength * repl); Assert.assertTrue(fs.delete(dir, true)); }
@Test public void testConstructorEmpty() { ContentSummary contentSummary = new ContentSummary.Builder().build(); assertEquals("getLength", 0, contentSummary.getLength()); assertEquals("getFileCount", 0, contentSummary.getFileCount()); assertEquals("getDirectoryCount", 0, contentSummary.getDirectoryCount()); assertEquals("getQuota", -1, contentSummary.getQuota()); assertEquals("getSpaceConsumed", 0, contentSummary.getSpaceConsumed()); assertEquals("getSpaceQuota", -1, contentSummary.getSpaceQuota()); }
@Test public void testConstructorEmpty() { ContentSummary contentSummary = new ContentSummary.Builder().build(); assertEquals("getLength", 0, contentSummary.getLength()); assertEquals("getFileCount", 0, contentSummary.getFileCount()); assertEquals("getDirectoryCount", 0, contentSummary.getDirectoryCount()); assertEquals("getQuota", -1, contentSummary.getQuota()); assertEquals("getSpaceConsumed", 0, contentSummary.getSpaceConsumed()); assertEquals("getSpaceQuota", -1, contentSummary.getSpaceQuota()); }
public static ContentSummaryProto convert(ContentSummary cs) { if (cs == null) return null; ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder(); builder.setLength(cs.getLength()). setFileCount(cs.getFileCount()). setDirectoryCount(cs.getDirectoryCount()). setSnapshotLength(cs.getSnapshotLength()). setSnapshotFileCount(cs.getSnapshotFileCount()). setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()). setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()). setQuota(cs.getQuota()). setSpaceConsumed(cs.getSpaceConsumed()). setSpaceQuota(cs.getSpaceQuota()). setErasureCodingPolicy(cs.getErasureCodingPolicy()); if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) { builder.setTypeQuotaInfos(getBuilder(cs)); } return builder.build(); }
@Test public void testConstructorNoQuota() { long length = 11111; long fileCount = 22222; long directoryCount = 33333; ContentSummary contentSummary = new ContentSummary.Builder().length(length). fileCount(fileCount).directoryCount(directoryCount). spaceConsumed(length).build(); assertEquals("getLength", length, contentSummary.getLength()); assertEquals("getFileCount", fileCount, contentSummary.getFileCount()); assertEquals("getDirectoryCount", directoryCount, contentSummary.getDirectoryCount()); assertEquals("getQuota", -1, contentSummary.getQuota()); assertEquals("getSpaceConsumed", length, contentSummary.getSpaceConsumed()); assertEquals("getSpaceQuota", -1, contentSummary.getSpaceQuota()); }
@Test public void testConstructorNoQuota() { long length = 11111; long fileCount = 22222; long directoryCount = 33333; ContentSummary contentSummary = new ContentSummary.Builder().length(length). fileCount(fileCount).directoryCount(directoryCount). spaceConsumed(length).build(); assertEquals("getLength", length, contentSummary.getLength()); assertEquals("getFileCount", fileCount, contentSummary.getFileCount()); assertEquals("getDirectoryCount", directoryCount, contentSummary.getDirectoryCount()); assertEquals("getQuota", -1, contentSummary.getQuota()); assertEquals("getSpaceConsumed", length, contentSummary.getSpaceConsumed()); assertEquals("getSpaceQuota", -1, contentSummary.getSpaceQuota()); }
@Test public void testConstructorWithQuota() { long length = 11111; long fileCount = 22222; long directoryCount = 33333; long quota = 44444; long spaceConsumed = 55555; long spaceQuota = 66666; ContentSummary contentSummary = new ContentSummary.Builder().length(length). fileCount(fileCount).directoryCount(directoryCount).quota(quota). spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).build(); assertEquals("getLength", length, contentSummary.getLength()); assertEquals("getFileCount", fileCount, contentSummary.getFileCount()); assertEquals("getDirectoryCount", directoryCount, contentSummary.getDirectoryCount()); assertEquals("getQuota", quota, contentSummary.getQuota()); assertEquals("getSpaceConsumed", spaceConsumed, contentSummary.getSpaceConsumed()); assertEquals("getSpaceQuota", spaceQuota, contentSummary.getSpaceQuota()); }