public CacheReplicationMonitor(FSNamesystem namesystem, CacheManager cacheManager, long intervalMs, ReentrantLock lock) { this.namesystem = namesystem; this.blockManager = namesystem.getBlockManager(); this.cacheManager = cacheManager; this.cachedBlocks = cacheManager.getCachedBlocks(); this.intervalMs = intervalMs; this.lock = lock; this.doRescan = this.lock.newCondition(); this.scanFinished = this.lock.newCondition(); }
this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(conf, dir); this.cacheManager = new CacheManager(this, conf, blockManager);
public void loadState(PersistState s) throws IOException { nextDirectiveId = s.section.getNextDirectiveId(); for (CachePoolInfoProto p : s.pools) { CachePoolInfo info = new CachePoolInfo(p.getPoolName()); if (p.hasOwnerName()) info.setOwnerName(p.getOwnerName()); if (p.hasGroupName()) info.setGroupName(p.getGroupName()); if (p.hasMode()) info.setMode(new FsPermission((short) p.getMode())); if (p.hasDefaultReplication()) { info.setDefaultReplication((short) p.getDefaultReplication()); } if (p.hasLimit()) info.setLimit(p.getLimit()); addCachePool(info); } for (CacheDirectiveInfoProto p : s.directives) { // Get pool reference by looking it up in the map final String poolName = p.getPool(); CacheDirective directive = new CacheDirective(p.getId(), new Path( p.getPath()).toUri().getPath(), (short) p.getReplication(), p .getExpiration().getMillis()); addCacheDirective(poolName, directive); } }
public void removeDirective(long id, FSPermissionChecker pc) throws IOException { assert namesystem.hasWriteLock(); try { CacheDirective directive = getById(id); checkWritePermission(pc, directive.getPool()); removeInternal(directive); } catch (IOException e) { LOG.warn("removeDirective of " + id + " failed: ", e); throw e; } LOG.info("removeDirective of " + id + " successful."); }
/** * Modifies a directive, skipping most error checking. This is for careful * internal use only. modifyDirective can be non-deterministic since its error * checking depends on current system time, which poses a problem for edit log * replay. */ void modifyDirectiveFromEditLog(CacheDirectiveInfo info) throws InvalidRequestException { // Check for invalid IDs. Long id = info.getId(); if (id == null) { throw new InvalidRequestException("Must supply an ID."); } CacheDirective prevEntry = getById(id); CacheDirectiveInfo newInfo = createFromInfoAndDefaults(info, prevEntry); removeInternal(prevEntry); addInternal(new CacheDirective(newInfo), getCachePool(newInfo.getPool())); }
public CacheDirectiveInfo addDirective( CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet<CacheFlag> flags) throws IOException { assert namesystem.hasWriteLock(); CacheDirective directive; try { CachePool pool = getCachePool(validatePoolName(info)); checkWritePermission(pc, pool); String path = validatePath(info); short replication = validateReplication( info, pool.getDefaultReplication()); long expiryTime = validateExpiryTime(info, pool.getMaxRelativeExpiryMs()); // Do quota validation if required if (!flags.contains(CacheFlag.FORCE)) { checkLimit(pool, path, replication); } // All validation passed // Add a new entry with the next available ID. long id = getNextDirectiveId(); directive = new CacheDirective(id, path, replication, expiryTime); addInternal(directive, pool); } catch (IOException e) { LOG.warn("addDirective of " + info + " failed: ", e); throw e; } LOG.info("addDirective of {} successful.", info); return directive.toInfo(); }
throw new InvalidRequestException("Must supply an ID."); CacheDirective prevEntry = getById(id); checkWritePermission(pc, prevEntry.getPool()); createFromInfoAndDefaults(info, prevEntry); CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(infoWithDefaults); validatePath(infoWithDefaults); validateReplication(infoWithDefaults, (short)-1); CachePool destPool = getCachePool(validatePoolName(infoWithDefaults)); if (!srcPool.getPoolName().equals(destPool.getPoolName())) { checkWritePermission(pc, destPool); if (!flags.contains(CacheFlag.FORCE)) { checkLimit(destPool, infoWithDefaults.getPath().toUri().getPath(), infoWithDefaults.getReplication()); validateExpiryTime(infoWithDefaults, destPool.getMaxRelativeExpiryMs()); setNeedsRescan(); removeInternal(prevEntry); addInternal(new CacheDirective(builder.build()), destPool); } catch (IOException e) { LOG.warn("modifyDirective of " + idString + " failed: ", e);
AddCacheDirectiveInfoOp addOp = (AddCacheDirectiveInfoOp) op; CacheDirectiveInfo result = fsNamesys. getCacheManager().addDirectiveFromEditLog(addOp.directive); if (toAddRetryCache) { Long id = result.getId(); ModifyCacheDirectiveInfoOp modifyOp = (ModifyCacheDirectiveInfoOp) op; fsNamesys.getCacheManager().modifyDirectiveFromEditLog( modifyOp.directive); if (toAddRetryCache) { RemoveCacheDirectiveInfoOp removeOp = (RemoveCacheDirectiveInfoOp) op; fsNamesys.getCacheManager().removeDirective(removeOp.id, null); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); fsNamesys.getCacheManager().addCachePool(addOp.info); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); fsNamesys.getCacheManager().modifyCachePool(modifyOp.info); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); fsNamesys.getCacheManager().removeCachePool(removeOp.poolName); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
static CachePoolInfo addCachePool( FSNamesystem fsn, CacheManager cacheManager, CachePoolInfo req, boolean logRetryCache) throws IOException { final FSPermissionChecker pc = getFsPermissionChecker(fsn); if (pc != null) { pc.checkSuperuserPrivilege(); } CachePoolInfo info = cacheManager.addCachePool(req); fsn.getEditLog().logAddCachePool(info, logRetryCache); return info; }
/** * Clear all loaded data */ void clear() { dir.reset(); dtSecretManager.reset(); leaseManager.removeAllLeases(); snapshotManager.clearSnapshottableDirs(); cacheManager.clear(); setImageLoaded(false); blockManager.clear(); ErasureCodingPolicyManager.getInstance().clear(); }
/** * Adds a directive, skipping most error checking. This should only be called * internally in special scenarios like edit log replay. */ CacheDirectiveInfo addDirectiveFromEditLog(CacheDirectiveInfo directive) throws InvalidRequestException { long id = directive.getId(); CacheDirective entry = new CacheDirective(directive); CachePool pool = cachePools.get(directive.getPool()); addInternal(entry, pool); if (nextDirectiveId <= id) { nextDirectiveId = id + 1; } return entry.toInfo(); }
static CacheDirectiveInfo addCacheDirective( FSNamesystem fsn, CacheManager cacheManager, CacheDirectiveInfo directive, EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException { final FSPermissionChecker pc = getFsPermissionChecker(fsn); if (directive.getId() != null) { throw new IOException("addDirective: you cannot specify an ID " + "for this operation."); } CacheDirectiveInfo effectiveDirective = cacheManager.addDirective(directive, pc, flags); fsn.getEditLog().logAddCacheDirectiveInfo(effectiveDirective, logRetryCache); return effectiveDirective; }
throw new InvalidRequestException("Must supply an ID."); CacheDirective prevEntry = getById(id); checkWritePermission(pc, prevEntry.getPool()); createFromInfoAndDefaults(info, prevEntry); CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(infoWithDefaults); validatePath(infoWithDefaults); validateReplication(infoWithDefaults, (short)-1); CachePool destPool = getCachePool(validatePoolName(infoWithDefaults)); if (!srcPool.getPoolName().equals(destPool.getPoolName())) { checkWritePermission(pc, destPool); if (!flags.contains(CacheFlag.FORCE)) { checkLimit(destPool, infoWithDefaults.getPath().toUri().getPath(), infoWithDefaults.getReplication()); validateExpiryTime(infoWithDefaults, destPool.getMaxRelativeExpiryMs()); setNeedsRescan(); removeInternal(prevEntry); addInternal(new CacheDirective(builder.build()), destPool); } catch (IOException e) { LOG.warn("modifyDirective of " + idString + " failed: ", e);
public CacheDirectiveInfo addDirective( CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet<CacheFlag> flags) throws IOException { assert namesystem.hasWriteLock(); CacheDirective directive; try { CachePool pool = getCachePool(validatePoolName(info)); checkWritePermission(pc, pool); String path = validatePath(info); short replication = validateReplication(info, (short)1); long expiryTime = validateExpiryTime(info, pool.getMaxRelativeExpiryMs()); // Do quota validation if required if (!flags.contains(CacheFlag.FORCE)) { checkLimit(pool, path, replication); } // All validation passed // Add a new entry with the next available ID. long id = getNextDirectiveId(); directive = new CacheDirective(id, path, replication, expiryTime); addInternal(directive, pool); } catch (IOException e) { LOG.warn("addDirective of " + info + " failed: ", e); throw e; } LOG.info("addDirective of {} successful.", info); return directive.toInfo(); }
AddCacheDirectiveInfoOp addOp = (AddCacheDirectiveInfoOp) op; CacheDirectiveInfo result = fsNamesys. getCacheManager().addDirectiveFromEditLog(addOp.directive); if (toAddRetryCache) { Long id = result.getId(); ModifyCacheDirectiveInfoOp modifyOp = (ModifyCacheDirectiveInfoOp) op; fsNamesys.getCacheManager().modifyDirectiveFromEditLog( modifyOp.directive); if (toAddRetryCache) { RemoveCacheDirectiveInfoOp removeOp = (RemoveCacheDirectiveInfoOp) op; fsNamesys.getCacheManager().removeDirective(removeOp.id, null); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); fsNamesys.getCacheManager().addCachePool(addOp.info); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); fsNamesys.getCacheManager().modifyCachePool(modifyOp.info); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); fsNamesys.getCacheManager().removeCachePool(removeOp.poolName); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
/** * Modifies a directive, skipping most error checking. This is for careful * internal use only. modifyDirective can be non-deterministic since its error * checking depends on current system time, which poses a problem for edit log * replay. */ void modifyDirectiveFromEditLog(CacheDirectiveInfo info) throws InvalidRequestException { // Check for invalid IDs. Long id = info.getId(); if (id == null) { throw new InvalidRequestException("Must supply an ID."); } CacheDirective prevEntry = getById(id); CacheDirectiveInfo newInfo = createFromInfoAndDefaults(info, prevEntry); removeInternal(prevEntry); addInternal(new CacheDirective(newInfo), getCachePool(newInfo.getPool())); }
public void removeDirective(long id, FSPermissionChecker pc) throws IOException { assert namesystem.hasWriteLock(); try { CacheDirective directive = getById(id); checkWritePermission(pc, directive.getPool()); removeInternal(directive); } catch (IOException e) { LOG.warn("removeDirective of " + id + " failed: ", e); throw e; } LOG.info("removeDirective of " + id + " successful."); }
/** * Load cache pools from fsimage */ private void loadPools(DataInput in) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.CACHE_POOLS); prog.beginStep(Phase.LOADING_FSIMAGE, step); int numberOfPools = in.readInt(); prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfPools); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step); for (int i = 0; i < numberOfPools; i++) { addCachePool(FSImageSerialization.readCachePoolInfo(in)); counter.increment(); } prog.endStep(Phase.LOADING_FSIMAGE, step); }
/** * Clear all loaded data */ void clear() { dir.reset(); dtSecretManager.reset(); blockIdManager.clear(); leaseManager.removeAllLeases(); snapshotManager.clearSnapshottableDirs(); cacheManager.clear(); setImageLoaded(false); blockManager.clear(); }
/** * Adds a directive, skipping most error checking. This should only be called * internally in special scenarios like edit log replay. */ CacheDirectiveInfo addDirectiveFromEditLog(CacheDirectiveInfo directive) throws InvalidRequestException { long id = directive.getId(); CacheDirective entry = new CacheDirective(directive); CachePool pool = cachePools.get(directive.getPool()); addInternal(entry, pool); if (nextDirectiveId <= id) { nextDirectiveId = id + 1; } return entry.toInfo(); }