private long waitForReplication() throws Exception { for (int i = 0; i < 10; i++) { long ur = cluster.getNamesystem().getUnderReplicatedBlocks(); if (ur == 0) { return 0; } else { Thread.sleep(1000); } } return cluster.getNamesystem().getUnderReplicatedBlocks(); }
/** @see ClientProtocol#getStats() */ long[] getStats() { final long[] stats = datanodeStatistics.getStats(); stats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = getUnderReplicatedBlocks(); stats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = getCorruptReplicaBlocks(); stats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = getMissingBlocksCount(); stats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] = getMissingReplOneBlocksCount(); return stats; }
/** @see ClientProtocol#getStats() */ long[] getStats() { final long[] stats = datanodeStatistics.getStats(); stats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = getUnderReplicatedBlocks(); stats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = getCorruptReplicaBlocks(); stats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = getMissingBlocksCount(); stats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] = getMissingReplOneBlocksCount(); return stats; }
private void printStats() { NameNodeAdapter.refreshBlockCounts(cluster.getNameNode()); if(LOG.isDebugEnabled()) { LOG.debug("Missing " + cluster.getNamesystem().getMissingBlocksCount()); LOG.debug("Corrupted " + cluster.getNamesystem().getCorruptReplicaBlocks()); LOG.debug("Under-replicated " + cluster.getNamesystem(). getUnderReplicatedBlocks()); LOG.debug("Pending delete " + cluster.getNamesystem(). getPendingDeletionBlocks()); LOG.debug("Pending replications " + cluster.getNamesystem(). getPendingReplicationBlocks()); LOG.debug("Excess " + cluster.getNamesystem().getExcessBlocks()); LOG.debug("Total " + cluster.getNamesystem().getBlocksTotal()); } }
long underRepl = fsn.getUnderReplicatedBlocks(); long pendRepl = fsn.getPendingReplicationBlocks(); long totalRepl = underRepl + pendRepl;
protected void startPostSafeModeProcessing() { // if not done yet, initialize replication queues if (!namesystem.isPopulatingReplQueues()) { initializeReplQueues(); } long timeInSafemode = FSNamesystem.now() - namesystem.systemStart; NameNode.stateChangeLog.info("STATE* Leaving safe mode after " + timeInSafemode / 1000 + " secs."); NameNode.getNameNodeMetrics().safeModeTime.set((int) timeInSafemode); if (reached >= 0) { NameNode.stateChangeLog.info("STATE* Safe mode is OFF."); } reached = -1; try { nameNode.startServerForClientRequests(); } catch (IOException ex) { nameNode.stop(); } NameNode.stateChangeLog.info("STATE* Network topology has " + namesystem.clusterMap.getNumOfRacks() + " racks and " + namesystem.clusterMap.getNumOfLeaves() + " datanodes"); NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has " + namesystem.getUnderReplicatedBlocks() + " blocks"); }
private LocatedBlocks createFileGetBlocks(String filenamePrefix) throws IOException { Path filePath = new Path("/" + filenamePrefix + ".dat"); // Write out a file with a few blocks, get block locations. DFSTestUtil.createFile(fs, filePath, BLOCK_SIZE, BLOCK_SIZE * NUM_BLOCKS, BLOCK_SIZE, NUM_DATANODES, seed); // Get the block list for the file with the block locations. LocatedBlocks blocks = client.getLocatedBlocks( filePath.toString(), 0, BLOCK_SIZE * NUM_BLOCKS); assertThat(cluster.getNamesystem().getUnderReplicatedBlocks(), is(0L)); return blocks; }
blocks2Remove.size(), cluster.getNamesystem().getMissingBlocksCount()); assertEquals("Wrong number of UnderReplicatedBlocks is found", blocks2Remove.size(), cluster.getNamesystem().getUnderReplicatedBlocks());
/** * Test creates a file and closes it. * The second datanode is started in the cluster. * As soon as the replication process is completed test runs * Block report and checks that no underreplicated blocks are left * * @throws IOException in case of an error */ @Test public void blockReport_06() throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); cluster.getNameNode().blockReport( cluster.getDataNodes().get(DN_N1).dnRegistration, new BlockListAsLongs(blocks, null).getBlockListAsLongs()); printStats(); assertEquals("Wrong number of PendingReplication Blocks", 0, cluster.getNamesystem().getUnderReplicatedBlocks()); }
private void printStats() { BlockManagerTestUtil.updateState(cluster.getNamesystem().getBlockManager()); if(LOG.isDebugEnabled()) { LOG.debug("Missing " + cluster.getNamesystem().getMissingBlocksCount()); LOG.debug("Corrupted " + cluster.getNamesystem().getCorruptReplicaBlocks()); LOG.debug("Under-replicated " + cluster.getNamesystem(). getUnderReplicatedBlocks()); LOG.debug("Pending delete " + cluster.getNamesystem(). getPendingDeletionBlocks()); LOG.debug("Pending replications " + cluster.getNamesystem(). getPendingReplicationBlocks()); LOG.debug("Excess " + cluster.getNamesystem().getExcessBlocks()); LOG.debug("Total " + cluster.getNamesystem().getBlocksTotal()); } }
long underRepl = fsn.getUnderReplicatedBlocks(); long pendRepl = fsn.getPendingReplicationBlocks(); long totalRepl = underRepl + pendRepl;
blocks2Remove.size(), cluster.getNamesystem().getMissingBlocksCount()); assertEquals("Wrong number of UnderReplicatedBlocks is found", blocks2Remove.size(), cluster.getNamesystem().getUnderReplicatedBlocks());
/** * Test creates a file and closes it. * The second datanode is started in the cluster. * As soon as the replication process is completed test runs * Block report and checks that no underreplicated blocks are left * * @throws IOException in case of an error */ @Test(timeout=300000) public void blockReport_06() throws Exception { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); sendBlockReports(dnR, poolId, reports); printStats(); assertEquals("Wrong number of PendingReplication Blocks", 0, cluster.getNamesystem().getUnderReplicatedBlocks()); }
long underReplicatedBlocks = nn.getNamesystem().getUnderReplicatedBlocks(); while (underReplicatedBlocks != (15 - safe)) { LOG.info("UnderReplicatedBlocks expected=" + (15 - safe) + Thread.sleep(100); BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager()); underReplicatedBlocks = nn.getNamesystem().getUnderReplicatedBlocks();
assertEquals(5, nn0.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0, nn1.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0, nn0.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0, nn1.getNamesystem().getUnderReplicatedBlocks());
assertEquals(0L, nn1.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0L, nn1.getNamesystem().getPendingReplicationBlocks());
pendingReplicationBlocks.pushMetric(metricsRecord); underReplicatedBlocks.set((int)fsNameSystem.getUnderReplicatedBlocks()); underReplicatedBlocks.pushMetric(metricsRecord);
pendingReplicationBlocks.set((int)fsNameSystem. getPendingReplicationBlocks()); underReplicatedBlocks.set((int)fsNameSystem.getUnderReplicatedBlocks()); scheduledReplicationBlocks.set((int)fsNameSystem. getScheduledReplicationBlocks());
HATestUtil.waitForDNDeletions(cluster); cluster.triggerDeletionReports(); assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks());
HATestUtil.waitForDNDeletions(cluster); cluster.triggerDeletionReports(); assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks());