m.put("ipAddr", datanodeinfo.getIpAddr()); m.put("name", datanodeinfo.getXferAddr()); m.put("hostName", datanodeinfo.getHostName()); m.put("storageID", datanodeinfo.getDatanodeUuid()); m.put("xferPort", datanodeinfo.getXferPort()); m.put("infoPort", datanodeinfo.getInfoPort()); m.put("infoSecurePort", datanodeinfo.getInfoSecurePort()); m.put("ipcPort", datanodeinfo.getIpcPort()); m.put("capacity", datanodeinfo.getCapacity()); m.put("dfsUsed", datanodeinfo.getDfsUsed()); m.put("remaining", datanodeinfo.getRemaining()); m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed()); m.put("cacheCapacity", datanodeinfo.getCacheCapacity()); m.put("cacheUsed", datanodeinfo.getCacheUsed()); m.put("lastUpdate", datanodeinfo.getLastUpdate()); m.put("lastUpdateMonotonic", datanodeinfo.getLastUpdateMonotonic()); m.put("xceiverCount", datanodeinfo.getXceiverCount()); m.put("networkLocation", datanodeinfo.getNetworkLocation()); m.put("adminState", datanodeinfo.getAdminState().name()); if (datanodeinfo.getUpgradeDomain() != null) { m.put("upgradeDomain", datanodeinfo.getUpgradeDomain()); m.put("lastBlockReportTime", datanodeinfo.getLastBlockReportTime()); m.put("lastBlockReportMonotonic", datanodeinfo.getLastBlockReportMonotonic()); return m;
@Test public void testLogRollOnDatanodeDeath() throws IOException, InterruptedException { dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), 3, true, null, null); tableName = getName(); Table table = createTestTable(tableName); TEST_UTIL.waitUntilAllRegionsAssigned(table.getName()); doPut(table, 1); server = TEST_UTIL.getRSForFirstRegionInTable(table.getName()); RegionInfo hri = server.getRegions(table.getName()).get(0).getRegionInfo(); AsyncFSWAL wal = (AsyncFSWAL) server.getWAL(hri); int numRolledLogFiles = AsyncFSWALProvider.getNumRolledLogFiles(wal); DatanodeInfo[] dnInfos = wal.getPipeline(); DataNodeProperties dnProp = TEST_UTIL.getDFSCluster().stopDataNode(dnInfos[0].getName()); TEST_UTIL.getDFSCluster().restartDataNode(dnProp); doPut(table, 2); assertEquals(numRolledLogFiles + 1, AsyncFSWALProvider.getNumRolledLogFiles(wal)); } }
private static void printDataNodeReports(DistributedFileSystem dfs, DatanodeReportType type, boolean listNodes, String nodeState) throws IOException { DatanodeInfo[] nodes = dfs.getDataNodeStats(type); if (nodes.length > 0 || listNodes) { System.out.println(nodeState + " datanodes (" + nodes.length + "):\n"); } if (nodes.length > 0) { for (DatanodeInfo dn : nodes) { System.out.println(dn.getDatanodeReport()); System.out.println(); } } }
private boolean isInactive(DatanodeInfo datanode) { return datanode.isDecommissioned() || (avoidStaleDataNodesForRead && datanode.isStale(staleInterval)); }
@Override public int compare(DatanodeInfo a, DatanodeInfo b) { // Decommissioned nodes will still be moved to the end of the list if (a.isDecommissioned()) { return b.isDecommissioned() ? 0 : 1; } else if (b.isDecommissioned()) { return -1; } // ENTERING_MAINTENANCE nodes should be after live nodes. if (a.isEnteringMaintenance()) { return b.isEnteringMaintenance() ? 0 : 1; } else if (b.isEnteringMaintenance()) { return -1; } else { return 0; } } }
/** * Match is checked using host name , ip address with and without port * number. * * @return true if the datanode's transfer address matches the set of nodes. */ private static boolean isIn(Set<String> datanodes, DatanodeInfo dn) { return isIn(datanodes, dn.getPeerHostName(), dn.getXferPort()) || isIn(datanodes, dn.getIpAddr(), dn.getXferPort()) || isIn(datanodes, dn.getHostName(), dn.getXferPort()); }
public DatanodeInfo(DatanodeInfo from) { super(from); this.capacity = from.getCapacity(); this.dfsUsed = from.getDfsUsed(); this.nonDfsUsed = from.getNonDfsUsed(); this.remaining = from.getRemaining(); this.blockPoolUsed = from.getBlockPoolUsed(); this.cacheCapacity = from.getCacheCapacity(); this.cacheUsed = from.getCacheUsed(); this.lastUpdate = from.getLastUpdate(); this.lastUpdateMonotonic = from.getLastUpdateMonotonic(); this.xceiverCount = from.getXceiverCount(); this.location = from.getNetworkLocation(); this.adminState = from.getAdminState(); }
protected DatanodeInfo(DatanodeInfo from) { super(from); this.capacity = from.getCapacity(); this.dfsUsed = from.getDfsUsed(); this.nonDfsUsed = from.getNonDfsUsed(); this.remaining = from.getRemaining(); this.blockPoolUsed = from.getBlockPoolUsed(); this.cacheCapacity = from.getCacheCapacity(); this.cacheUsed = from.getCacheUsed(); this.lastUpdate = from.getLastUpdate(); this.lastUpdateMonotonic = from.getLastUpdateMonotonic(); this.xceiverCount = from.getXceiverCount(); this.location = from.getNetworkLocation(); this.adminState = from.getAdminState(); this.upgradeDomain = from.getUpgradeDomain(); this.lastBlockReportTime = from.getLastBlockReportTime(); this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); this.numBlocks = from.getNumBlocks(); }
long c = getCapacity(); long r = getRemaining(); long u = getDfsUsed(); long nonDFSUsed = getNonDfsUsed(); float usedPercent = getDfsUsedPercent(); float remainingPercent = getRemainingPercent(); long cc = getCacheCapacity(); long cr = getCacheRemaining(); long cu = getCacheUsed(); float cacheUsedPercent = getCacheUsedPercent(); float cacheRemainingPercent = getCacheRemainingPercent(); String lookupName = NetUtils.getHostNameOfIP(getName()); buffer.append("Name: "+ getName()); if (lookupName != null) { buffer.append(" (" + lookupName + ")"); buffer.append("Hostname: " + getHostName() + "\n"); if (isDecommissioned()) { buffer.append("Decommissioned\n"); } else if (isDecommissionInProgress()) { buffer.append("Decommission in progress\n"); } else { buffer.append("Cache Used%: "+percent2String(cacheUsedPercent) + "\n"); buffer.append("Cache Remaining%: "+percent2String(cacheRemainingPercent) + "\n"); buffer.append("Xceivers: "+getXceiverCount()+"\n"); buffer.append("Last contact: "+new Date(lastUpdate)+"\n");
private void compare(DatanodeInfo dn1, DatanodeInfo dn2) { assertEquals(dn1.getAdminState(), dn2.getAdminState()); assertEquals(dn1.getBlockPoolUsed(), dn2.getBlockPoolUsed()); assertEquals(dn1.getBlockPoolUsedPercent(), dn2.getBlockPoolUsedPercent(), DELTA); assertEquals(dn1.getCapacity(), dn2.getCapacity()); assertEquals(dn1.getDatanodeReport(), dn2.getDatanodeReport()); assertEquals(dn1.getDfsUsed(), dn1.getDfsUsed()); assertEquals(dn1.getDfsUsedPercent(), dn1.getDfsUsedPercent(), DELTA); assertEquals(dn1.getIpAddr(), dn2.getIpAddr()); assertEquals(dn1.getHostName(), dn2.getHostName()); assertEquals(dn1.getInfoPort(), dn2.getInfoPort()); assertEquals(dn1.getIpcPort(), dn2.getIpcPort()); assertEquals(dn1.getLastUpdate(), dn2.getLastUpdate()); assertEquals(dn1.getLevel(), dn2.getLevel()); assertEquals(dn1.getNetworkLocation(), dn2.getNetworkLocation()); }
public DatanodeInfo(DatanodeInfo from) { super(from); this.capacity = from.getCapacity(); this.dfsUsed = from.getDfsUsed(); this.remaining = from.getRemaining(); this.namespaceUsed = from.getNamespaceUsed(); this.lastUpdate = from.getLastUpdate(); this.xceiverCount = from.getXceiverCount(); this.location = from.getNetworkLocation(); this.adminState = from.adminState; this.hostName = from.hostName; }
/** A formatted string for printing the status of the DataNode. */ public String dumpDatanode() { StringBuffer buffer = new StringBuffer(); long c = getCapacity(); long r = getRemaining(); long u = getDfsUsed(); buffer.append(name); if (!NetworkTopology.DEFAULT_RACK.equals(location)) { buffer.append(" "+location); } if (isDecommissioned()) { buffer.append(" DD"); } else if (isDecommissionInProgress()) { buffer.append(" DP"); } else { buffer.append(" IN"); } buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")"); buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")"); buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%"); buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")"); buffer.append(" " + new Date(lastUpdate)); return buffer.toString(); }
boolean found = false; for (int i = 0; i < dnis.length - 1 && !found; i++) { if (hostName.equals(dnis[i].getHostName())) {
/** * This function maps the required fields from DataNodeInfo to disk * BalancerDataNode. * * @param nodeInfo * @return DiskBalancerDataNode */ private DiskBalancerDataNode getBalancerNodeFromDataNode(DatanodeInfo nodeInfo) { Preconditions.checkNotNull(nodeInfo); DiskBalancerDataNode dbDataNode = new DiskBalancerDataNode(nodeInfo .getDatanodeUuid()); dbDataNode.setDataNodeIP(nodeInfo.getIpAddr()); dbDataNode.setDataNodeName(nodeInfo.getHostName()); dbDataNode.setDataNodePort(nodeInfo.getIpcPort()); return dbDataNode; }
private boolean checkNodeState(FileSystem filesys, String node, NodeState state) throws IOException { DistributedFileSystem dfs = (DistributedFileSystem) filesys; boolean done = false; boolean foundNode = false; DatanodeInfo[] datanodes = dfs.getDataNodeStats(); for (int i = 0; i < datanodes.length; i++) { DatanodeInfo dn = datanodes[i]; if (dn.getName().equals(node)) { if (state == NodeState.DECOMMISSIONED) { done = dn.isDecommissioned(); } else if (state == NodeState.DECOMMISSION_INPROGRESS) { done = dn.isDecommissionInProgress(); } else { done = (!dn.isDecommissionInProgress() && !dn.isDecommissioned()); } System.out.println(dn.getDatanodeReport()); foundNode = true; } } if (!foundNode) { throw new IOException("Could not find node: " + node); } return done; }
private void verifyStats(NameNode namenode, FSNamesystem fsn, DatanodeInfo info, DataNode node, boolean decommissioning) throws InterruptedException, IOException { // Do the stats check over 10 heartbeats for (int i = 0; i < 10; i++) { long[] newStats = namenode.getRpcServer().getStats(); // For decommissioning nodes, ensure capacity of the DN is no longer // counted. Only used space of the DN is counted in cluster capacity assertEquals(newStats[0], decommissioning ? info.getDfsUsed() : info.getCapacity()); // Ensure cluster used capacity is counted for both normal and // decommissioning nodes assertEquals(newStats[1], info.getDfsUsed()); // For decommissioning nodes, remaining space from the DN is not counted assertEquals(newStats[2], decommissioning ? 0 : info.getRemaining()); // Ensure transceiver count is same as that DN assertEquals(fsn.getTotalLoad(), info.getXceiverCount()); DataNodeTestUtils.triggerHeartbeat(node); } }
public DatanodeInfo build() { return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed, remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic, xceiverCount, location, adminState, upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic, numBlocks); } }
static private double getRemaining(DatanodeInfo datanode) { return ((double)datanode.getRemaining())/datanode.getCapacity()*100; }
(prependReservedRawPath ? "/.reserved/raw" + path : path); int port = "http".equals(scheme) ? dn.getInfoPort() : dn .getInfoSecurePort(); final URI uri = new URI(scheme, null, dn.getHostName(), port, uripath, queryBuilder.toString(), null);
/** * Get rack string from a data node * @return rack of data node */ protected String getRack(final DatanodeInfo datanode) { return datanode.getNetworkLocation(); }