int remove(DatanodeDescriptor dnToRemove) { // this operation happens under the FSNamesystem lock; // no additional synchronization required. if (dnToRemove != null) { DatanodeDescriptor storedDN = dns.get(dnToRemove.getDatanodeUuid()); if (storedDN != null) { dns.remove(dnToRemove.getDatanodeUuid()); dnR.remove(dnToRemove); } } return dns.size(); }
public synchronized void unregister(DatanodeDescriptor dn) { NodeData node = nodes.remove(dn.getDatanodeUuid()); if (node == null) { LOG.info("Can't unregister DN {} because it is not currently " + "registered.", dn.getDatanodeUuid()); return; } remove(node); }
private synchronized NodeData registerNode(DatanodeDescriptor dn) { if (nodes.containsKey(dn.getDatanodeUuid())) { LOG.info("Can't register DN {} because it is already registered.", dn.getDatanodeUuid()); return null; } NodeData node = new NodeData(dn.getDatanodeUuid()); deferredHead.addToBeginning(node); nodes.put(dn.getDatanodeUuid(), node); LOG.info("Registered DN {} ({}).", dn.getDatanodeUuid(), dn.getXferAddr()); return node; }
DatanodeStorageInfo getProvidedStorage( DatanodeDescriptor dn, DatanodeStorage s) { dns.put(dn.getDatanodeUuid(), dn); dnR.add(dn); return storageMap.get(s.getStorageID()); }
/** * @return does this map contains a redundancy corresponding to the given * datanode and the given block? */ synchronized boolean contains(DatanodeDescriptor dn, BlockInfo blk) { final LightWeightHashSet<BlockInfo> set = map.get(dn.getDatanodeUuid()); return set != null && set.contains(blk); }
public synchronized long removeLease(DatanodeDescriptor dn) { NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { LOG.info("Can't remove lease for unknown datanode {}", dn.getDatanodeUuid()); return 0; } long id = node.leaseId; if (id == 0) { LOG.debug("DN {} has no lease to remove.", dn.getDatanodeUuid()); return 0; } remove(node); deferredHead.addToEnd(node); if (LOG.isTraceEnabled()) { LOG.trace("Removed BR lease 0x{} for DN {}. numPending = {}", Long.toHexString(id), dn.getDatanodeUuid(), numPending); } return id; } }
DatanodeDescriptor choose(DatanodeDescriptor client, Set<String> excludedUUids) { // exact match for now if (client != null && !excludedUUids.contains(client.getDatanodeUuid())) { DatanodeDescriptor dn = dns.get(client.getDatanodeUuid()); if (dn != null) { return dn; } } // prefer live nodes first. DatanodeDescriptor dn = chooseRandomNode(excludedUUids, true); if (dn == null) { dn = chooseRandomNode(excludedUUids, false); } return dn; }
private DatanodeDescriptor chooseRandomNode(Set<String> excludedUUids, boolean preferLiveNodes) { Random r = new Random(); for (int i = dnR.size() - 1; i >= 0; --i) { int pos = r.nextInt(i + 1); DatanodeDescriptor node = dnR.get(pos); String uuid = node.getDatanodeUuid(); if (!excludedUUids.contains(uuid)) { if (!preferLiveNodes || node.getAdminState() == AdminStates.NORMAL) { return node; } } Collections.swap(dnR, i, pos); } return null; }
if (id == 0) { LOG.debug("Datanode {} is using BR lease id 0x0 to bypass " + "rate-limiting.", dn.getDatanodeUuid()); return true; NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { LOG.info("BR lease 0x{} is not valid for unknown datanode {}", Long.toHexString(id), dn.getDatanodeUuid()); return false; LOG.warn("BR lease 0x{} is not valid for DN {}, because the DN " + "is not in the pending set.", Long.toHexString(id), dn.getDatanodeUuid()); return false; "has expired.", Long.toHexString(id), dn.getDatanodeUuid()); return false; Long.toHexString(id), dn.getDatanodeUuid(), Long.toHexString(node.leaseId)); return false; Long.toHexString(id), dn.getDatanodeUuid());
/** * Add the redundancy of the given block stored in the given datanode to the * map. * * @return true if the block is added. */ synchronized boolean add(DatanodeDescriptor dn, BlockInfo blk) { LightWeightHashSet<BlockInfo> set = map.get(dn.getDatanodeUuid()); if (set == null) { set = new LightWeightHashSet<>(); map.put(dn.getDatanodeUuid(), set); } final boolean added = set.add(blk); if (added) { size.incrementAndGet(); blockLog.debug("BLOCK* ExcessRedundancyMap.add({}, {})", dn, blk); } return added; }
DatanodeDescriptor chooseRandom(DatanodeStorageInfo... excludedStorages) { Set<String> excludedNodes = new HashSet<>(); if (excludedStorages != null) { for (int i = 0; i < excludedStorages.length; i++) { DatanodeDescriptor dn = excludedStorages[i].getDatanodeDescriptor(); String uuid = dn.getDatanodeUuid(); excludedNodes.add(uuid); } } return choose(null, excludedNodes); }
/** * Remove the redundancy corresponding to the given datanode and the given * block. * * @return true if the block is removed. */ synchronized boolean remove(DatanodeDescriptor dn, BlockInfo blk) { final LightWeightHashSet<BlockInfo> set = map.get(dn.getDatanodeUuid()); if (set == null) { return false; } final boolean removed = set.remove(blk); if (removed) { size.decrementAndGet(); blockLog.debug("BLOCK* ExcessRedundancyMap.remove({}, {})", dn, blk); if (set.isEmpty()) { map.remove(dn.getDatanodeUuid()); } } return removed; } }
public synchronized long requestLease(DatanodeDescriptor dn) { NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { LOG.warn("DN {} ({}) requested a lease even though it wasn't yet " + "registered. Registering now.", dn.getDatanodeUuid(), dn.getXferAddr()); node = registerNode(dn); dn.getDatanodeUuid()); dn.getDatanodeUuid(), numPending, allLeases.toString()); if (LOG.isDebugEnabled()) { LOG.debug("Created a new BR lease 0x{} for DN {}. numPending = {}", Long.toHexString(node.leaseId), dn.getDatanodeUuid(), numPending);
/** Add a datanode. */ void addDatanode(final DatanodeDescriptor node) { // To keep host2DatanodeMap consistent with datanodeMap, // remove from host2DatanodeMap the datanodeDescriptor removed // from datanodeMap before adding node to host2DatanodeMap. synchronized(this) { host2DatanodeMap.remove(datanodeMap.put(node.getDatanodeUuid(), node)); } networktopology.add(node); // may throw InvalidTopologyException host2DatanodeMap.add(node); checkIfClusterIsNowMultiRack(node); resolveUpgradeDomain(node); if (LOG.isDebugEnabled()) { LOG.debug(getClass().getSimpleName() + ".addDatanode: " + "node " + node + " is added to datanodeMap."); } }
currInfo.getDatanodeDescriptor(), currInfo.getStorageID(), storageType)); excludedUUids.add(currInfo.getDatanodeDescriptor().getDatanodeUuid()); locs.add( new DatanodeInfoWithStorage(dn, storageId, StorageType.PROVIDED)); excludedUUids.add(dn.getDatanodeUuid()); numLocations++; sids.add(storageId); types.add(StorageType.PROVIDED); excludedUUids.add(dn.getDatanodeUuid());
for(int i = 0; i < locations.size(); i++) { final DatanodeStorageInfo s = locations.get(i); datanodeUuids[i] = s.getDatanodeDescriptor().getDatanodeUuid(); storageIDs[i] = s.getStorageID(); storageTypes[i] = s.getStorageType();
double ratio = storage.treeSetFillRatio(); if (ratio < storageInfoDefragmentRatio) { datanodesAndStorages.add(node.getDatanodeUuid()); datanodesAndStorages.add(storage.getStorageID());
dn.getDatanodeUuid()); continue; LOG.debug("Block {}: removing from PENDING_CACHED for node {} " + "because it cannot fit in remaining cache size {}.", cblock.getBlockId(), dn.getDatanodeUuid(), remaining); it.remove(); } else { LOG.trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because the DataNode uncached it.", cblock.getBlockId(), datanode.getDatanodeUuid()); datanode.getPendingUncached().remove(cblock); iter.remove(); + "because we already have {} cached replicas and we only" + " need {}", cblock.getBlockId(), datanode.getDatanodeUuid(), numCached, neededCached ); LOG.trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because we only have {} cached replicas and we need " + "{}", cblock.getBlockId(), datanode.getDatanodeUuid(), numCached, neededCached );
+ "NameNode ({}).", strBlockReportId, storageInfo.getDatanodeDescriptor().getDatanodeUuid(), VersionInfo.getVersion()); Set<BlockReportReplica> set = new FoldedTreeSet<>();
"because the block has size {}, but the DataNode only has {} " + "bytes of cache remaining ({} pending bytes, {} already cached.)", blockInfo.getBlockId(), datanode.getDatanodeUuid(), blockInfo.getNumBytes(), pendingCapacity, pendingBytes, datanode.getCacheRemaining()); for (DatanodeDescriptor datanode : chosen) { LOG.trace("Block {}: added to PENDING_CACHED on DataNode {}", blockInfo.getBlockId(), datanode.getDatanodeUuid()); pendingCached.add(datanode); boolean added = datanode.getPendingCached().add(cachedBlock);