public Loader(FSNamesystem fsn, FSImageFormatProtobuf.Loader parent) { this.fsn = fsn; this.fsDir = fsn.getFSDirectory(); this.snapshotMap = new HashMap<Integer, Snapshot>(); this.parent = parent; }
@Override public INodeFile getBlockCollection(long id) { INode inode = getFSDirectory().getInode(id); return inode == null ? null : inode.asFile(); }
private String reconfProtectedDirectories(String newVal) { return getNamesystem().getFSDirectory().setProtectedDirectories(newVal); }
/** * Get the erasure coding policy, including the REPLICATION policy. This does * not do any permission checking. * * @param fsn namespace * @param iip inodes in the path containing the file * @return {@link ErasureCodingPolicy} * @throws IOException */ static ErasureCodingPolicy unprotectedGetErasureCodingPolicy( final FSNamesystem fsn, final INodesInPath iip) throws IOException { assert fsn.hasReadLock(); return getErasureCodingPolicyForPath(fsn.getFSDirectory(), iip); }
private synchronized INode[] getINodesWithLease() { List<INode> inodes = new ArrayList<>(leasesById.size()); INode currentINode; for (long inodeId : leasesById.keySet()) { currentINode = fsnamesystem.getFSDirectory().getInode(inodeId); // A file with an active lease could get deleted, or its // parent directories could get recursively deleted. if (currentINode != null && currentINode.isFile() && !fsnamesystem.isFileDeleted(currentINode.asFile())) { inodes.add(currentINode); } } return inodes.toArray(new INode[0]); }
final FSDirectory dir = fsn.getFSDirectory(); synchronized (leaseManager) { Collection<Long> filesWithUC = leaseManager.getINodeIdWithLeases();
@Override public List<INodesInPath> call() { List<INodesInPath> iNodesInPaths = Lists.newArrayList(); for (int idx = startIdx; idx < inodeCount; idx += workerCount) { INode inode = inodes[idx]; if (!inode.isFile()) { continue; } INodesInPath inodesInPath = INodesInPath.fromINode( fsnamesystem.getFSDirectory().getRoot(), inode.asFile()); if (ancestorDir != null && !inodesInPath.isDescendant(ancestorDir)) { continue; } iNodesInPaths.add(inodesInPath); } return iNodesInPaths; } };
private static List<XAttr> removeErasureCodingPolicyXAttr( final FSNamesystem fsn, final INodesInPath srcIIP) throws IOException { FSDirectory fsd = fsn.getFSDirectory(); assert fsd.hasWriteLock(); Preconditions.checkNotNull(srcIIP, "INodes cannot be null"); String src = srcIIP.getPath(); final INode inode = srcIIP.getLastINode(); if (inode == null) { throw new FileNotFoundException("Path not found: " + srcIIP.getPath()); } if (!inode.isDirectory()) { throw new IOException("Cannot unset an erasure coding policy " + "on a file " + src); } // Check whether the directory has a specific erasure coding policy // directly on itself. final XAttr ecXAttr = getErasureCodingPolicyXAttrForINode(fsn, inode); if (ecXAttr == null) { return null; } final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1); xattrs.add(ecXAttr); return FSDirXAttrOp.unprotectedRemoveXAttrs(fsd, srcIIP, xattrs); }
FSDirectory fsd = fsn.getFSDirectory(); final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ); if (fsn.isPermissionEnabled()) { fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
private LocatedBlocks getBlockLocations(String path, HdfsFileStatus file) throws IOException { long fileLen = file.getLen(); LocatedBlocks blocks = null; final FSNamesystem fsn = namenode.getNamesystem(); fsn.readLock(); try { blocks = FSDirStatAndListingOp.getBlockLocations( fsn.getFSDirectory(), fsn.getPermissionChecker(), path, 0, fileLen, false) .blocks; } catch (FileNotFoundException fnfe) { blocks = null; } finally { fsn.readUnlock("fsckGetBlockLocations"); } return blocks; }
private static XAttr getErasureCodingPolicyXAttrForINode( FSNamesystem fsn, INode inode) throws IOException { // INode can be null if (inode == null) { return null; } FSDirectory fsd = fsn.getFSDirectory(); fsd.readLock(); try { // We don't allow setting EC policies on paths with a symlink. Thus // if a symlink is encountered, the dir shouldn't have EC policy. // TODO: properly support symlinks if (inode.isSymlink()) { return null; } final XAttrFeature xaf = inode.getXAttrFeature(); if (xaf != null) { XAttr xattr = xaf.getXAttr(XATTR_ERASURECODING_POLICY); if (xattr != null) { return xattr; } } } finally { fsd.readUnlock(); } return null; } }
/** * save all the snapshot diff to fsimage */ public void serializeSnapshotDiffSection(OutputStream out) throws IOException { INodeMap inodesMap = fsn.getFSDirectory().getINodeMap(); final List<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator(); while (iter.hasNext()) { INodeWithAdditionalFields inode = iter.next(); if (inode.isFile()) { serializeFileDiffList(inode.asFile(), out); } else if (inode.isDirectory()) { serializeDirDiffList(inode.asDirectory(), refList, out); } ++i; if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { context.checkCancelled(); } } parent.commitSection(headers, FSImageFormatProtobuf.SectionName.SNAPSHOT_DIFF); }
void serializeFilesUCSection(OutputStream out) throws IOException { Collection<Long> filesWithUC = fsn.getLeaseManager() .getINodeIdWithLeases(); for (Long id : filesWithUC) { INode inode = fsn.getFSDirectory().getInode(id); if (inode == null) { LOG.warn("Fail to find inode " + id + " when saving the leases."); continue; } INodeFile file = inode.asFile(); if (!file.isUnderConstruction()) { LOG.warn("Fail to save the lease for inode id " + id + " as the file is not under construction"); continue; } String path = file.getFullPathName(); FileUnderConstructionEntry.Builder b = FileUnderConstructionEntry .newBuilder().setInodeId(file.getId()).setFullPath(path); FileUnderConstructionEntry e = b.build(); e.writeDelimitedTo(out); } parent.commitSection(summary, FSImageFormatProtobuf.SectionName.FILES_UNDERCONSTRUCTION); }
blockManager.getDatanodeManager().getDatanodes()) { for (long ucFileId : dataNode.getLeavingServiceStatus().getOpenFiles()) { INode ucFile = getFSDirectory().getInode(ucFileId); if (ucFile == null || ucFileId <= prevId || openFileIds.contains(ucFileId)) {
private INodeReferenceSection.INodeReference.Builder buildINodeReference( final INodeReference ref, final long refIndex) throws IOException { INodeReferenceSection.INodeReference.Builder rb = INodeReferenceSection.INodeReference.newBuilder(). setReferredId(ref.getId()); if (ref instanceof WithName) { rb.setLastSnapshotId(((WithName) ref).getLastSnapshotId()).setName( ByteString.copyFrom(ref.getLocalNameBytes())); } else if (ref instanceof DstReference) { rb.setDstSnapshotId(ref.getDstSnapshotId()); } if (fsn.getFSDirectory().getInode(ref.getId()) == null) { FSImage.LOG.error( "FSImageFormatPBSnapshot: Missing referred INodeId " + ref.getId() + " for INodeReference index " + refIndex + "; path=" + ref.getFullPathName() + "; parent=" + (ref.getParent() == null ? "null" : ref.getParent().getFullPathName())); ++numImageErrors; } return rb; }
/** * Verify quota when using the preferred block size for UC block. This is * usually used by append and truncate. * * @throws QuotaExceededException when violating the storage quota * @return expected quota usage update. null means no change or no need to * update quota usage later */ private static QuotaCounts verifyQuotaForUCBlock(FSNamesystem fsn, INodeFile file, INodesInPath iip) throws QuotaExceededException { FSDirectory fsd = fsn.getFSDirectory(); if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) { // Do not check quota if editlog is still being processed return null; } if (file.getLastBlock() != null) { final QuotaCounts delta = computeQuotaDeltaForUCBlock(fsn, file); fsd.readLock(); try { FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null); return delta; } finally { fsd.readUnlock(); } } return null; }
private static void verifyQuotaForTruncate(FSNamesystem fsn, INodesInPath iip, INodeFile file, long newLength, QuotaCounts delta) throws QuotaExceededException { FSDirectory fsd = fsn.getFSDirectory(); if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) { // Do not check quota if edit log is still being processed return; } final BlockStoragePolicy policy = fsd.getBlockStoragePolicySuite() .getPolicy(file.getStoragePolicyID()); file.computeQuotaDeltaForTruncate(newLength, policy, delta); fsd.readLock(); try { FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null); } finally { fsd.readUnlock(); } }
/** Compute quota change for converting a complete block to a UC block. */ private static QuotaCounts computeQuotaDeltaForUCBlock(FSNamesystem fsn, INodeFile file) { final QuotaCounts delta = new QuotaCounts.Builder().build(); final BlockInfo lastBlock = file.getLastBlock(); if (lastBlock != null) { final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes(); final short repl = lastBlock.getReplication(); delta.addStorageSpace(diff * repl); final BlockStoragePolicy policy = fsn.getFSDirectory() .getBlockStoragePolicySuite().getPolicy(file.getStoragePolicyID()); List<StorageType> types = policy.chooseStorageTypes(repl); for (StorageType t : types) { if (t.supportTypeQuota()) { delta.addTypeSpace(t, diff); } } } return delta; } }
FSDirectory fsd = fsn.getFSDirectory(); final INodesInPath iip; List<XAttr> xAttrs;
FSDirectory fsDir = namesystem.getFSDirectory(); INode node; long requestedBytes = 0;