cluster.startDataNodes(conf, 1, true, null, new String[]{"/r4"}, new String[]{host4}, null); cluster.waitClusterUp();
@Test public void testLogRollOnDatanodeDeath() throws IOException, InterruptedException { dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), 3, true, null, null); tableName = getName(); Table table = createTestTable(tableName); TEST_UTIL.waitUntilAllRegionsAssigned(table.getName()); doPut(table, 1); server = TEST_UTIL.getRSForFirstRegionInTable(table.getName()); RegionInfo hri = server.getRegions(table.getName()).get(0).getRegionInfo(); AsyncFSWAL wal = (AsyncFSWAL) server.getWAL(hri); int numRolledLogFiles = AsyncFSWALProvider.getNumRolledLogFiles(wal); DatanodeInfo[] dnInfos = wal.getPipeline(); DataNodeProperties dnProp = TEST_UTIL.getDFSCluster().stopDataNode(dnInfos[0].getName()); TEST_UTIL.getDFSCluster().restartDataNode(dnProp); doPut(table, 2); assertEquals(numRolledLogFiles + 1, AsyncFSWALProvider.getNumRolledLogFiles(wal)); } }
dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), numDataNodes, true, null, null); List<DataNode> allNodes = dfsCluster.getDataNodes(); for (int i = allNodes.size() - 1; i >= 0; i--) { !log.isLowReplicationRollEnabled()); dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), 1, true, null, null);
/** * Modify the config and start up the DataNodes. The info port for * DataNodes is guaranteed to use a free port. * * @param conf the base configuration to use in starting the DataNodes. This * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be * created and dfs.data.dir will be set in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * @param racks array of strings indicating the rack that each DataNode is on * * @throws IllegalStateException if NameNode has been shutdown */ public void startDataNodes(Configuration conf, int numDataNodes, boolean manageDfsDirs, StartupOption operation, String[] racks ) throws IOException { startDataNodes( conf, numDataNodes, manageDfsDirs, operation, racks, null, null); }
public synchronized void startDataNodes(Configuration conf, int numDataNodes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig) throws IOException { startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, null, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null); }
/** * Modify the config and start up the DataNodes. The info port for * DataNodes is guaranteed to use a free port. * * @param conf the base configuration to use in starting the DataNodes. This * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be * set in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * @param racks array of strings indicating the rack that each DataNode is on * * @throws IllegalStateException if NameNode has been shutdown */ public void startDataNodes(Configuration conf, int numDataNodes, boolean manageDfsDirs, StartupOption operation, String[] racks ) throws IOException { startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, null, null, false); }
/** * Attempts to start a DataNode with the given operation. Starting * the DataNode should throw an exception. */ void startDataNodeShouldFail(StartupOption operation) { try { cluster.startDataNodes(conf, 1, false, operation, null); // should fail throw new AssertionError("DataNode should have failed to start"); } catch (Exception expected) { // expected assertFalse(cluster.isDataNodeUp()); } }
/** * Attempts to start a DataNode with the given operation. Starting * the DataNode should throw an exception. */ void startDataNodeShouldFail(StartupOption operation) { try { cluster.startDataNodes(conf, 1, false, operation, null); // should fail throw new AssertionError("DataNode should have failed to start"); } catch (Exception expected) { // expected assertFalse(cluster.isDataNodeUp()); } }
/** * Attempts to start a DataNode with the given operation. Starting * the DataNode should throw an exception. */ void startDataNodeShouldFail(StartupOption operation) { try { cluster.startDataNodes(conf, 1, false, operation, null); // should fail throw new AssertionError("DataNode should have failed to start"); } catch (Exception expected) { // expected assertFalse(cluster.isDataNodeUp()); } }
private void startDNandWait(Path filePath, boolean waitReplicas) throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Before next DN start: " + cluster.getDataNodes().size()); } cluster.startDataNodes(conf, 1, true, null, null); ArrayList<DataNode> datanodes = cluster.getDataNodes(); assertEquals(datanodes.size(), 2); if(LOG.isDebugEnabled()) { LOG.debug("New datanode " + cluster.getDataNodes().get(datanodes.size() - 1) .getDatanodeRegistration() + " has been started"); } if (waitReplicas) DFSTestUtil.waitReplication(fs, filePath, REPL_FACTOR); }
/** * Restart the datanodes with a new volume tolerated value. * @param volTolerated number of dfs data dir failures to tolerate * @param manageDfsDirs whether the mini cluster should manage data dirs * @throws IOException */ private void restartDatanodes(int volTolerated, boolean manageDfsDirs) throws IOException { // Make sure no datanode is running cluster.shutdownDataNodes(); conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, volTolerated); cluster.startDataNodes(conf, 1, manageDfsDirs, null, null); cluster.waitActive(); }
/** * Attempts to start a DataNode with the given operation. Starting * the given block pool should fail. * @param operation startup option * @param bpid block pool Id that should fail to start * @throws IOException */ void startBlockPoolShouldFail(StartupOption operation, String bpid) throws IOException { cluster.startDataNodes(conf, 1, false, operation, null); // should fail assertFalse("Block pool " + bpid + " should have failed to start", cluster.getDataNodes().get(0).isBPServiceAlive(bpid)); }
/** * Attempts to start a DataNode with the given operation. Starting * the given block pool should fail. * @param operation startup option * @param bpid block pool Id that should fail to start * @throws IOException */ void startBlockPoolShouldFail(StartupOption operation, String bpid) throws IOException { cluster.startDataNodes(conf, 1, false, operation, null); // should fail assertFalse("Block pool " + bpid + " should have failed to start", cluster.getDataNodes().get(0).isBPServiceAlive(bpid)); }
private void startDNandWait(Path filePath, boolean waitReplicas) throws IOException, InterruptedException, TimeoutException { if (LOG.isDebugEnabled()) { LOG.debug("Before next DN start: " + cluster.getDataNodes().size()); } cluster.startDataNodes(conf, 1, true, null, null); cluster.waitClusterUp(); ArrayList<DataNode> datanodes = cluster.getDataNodes(); assertEquals(datanodes.size(), 2); if (LOG.isDebugEnabled()) { int lastDn = datanodes.size() - 1; LOG.debug("New datanode " + cluster.getDataNodes().get(lastDn).getDisplayName() + " has been started"); } if (waitReplicas) { DFSTestUtil.waitReplication(fs, filePath, REPL_FACTOR); } }
/** * Checks whether {@link DataNode#checkDiskErrorAsync()} is being called or not. * Before refactoring the code the above function was not getting called * @throws IOException, InterruptedException */ @Test public void testcheckDiskError() throws IOException, InterruptedException { if(cluster.getDataNodes().size() <= 0) { cluster.startDataNodes(conf, 1, true, null, null); cluster.waitActive(); } DataNode dataNode = cluster.getDataNodes().get(0); long slackTime = dataNode.checkDiskErrorInterval/2; //checking for disk error dataNode.checkDiskErrorAsync(); Thread.sleep(dataNode.checkDiskErrorInterval); long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck(); assertTrue("Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms", ((Time.monotonicNow()-lastDiskErrorCheck) < (dataNode.checkDiskErrorInterval + slackTime))); }
@Test public void testSufficientlyReplBlocksUsesNewRack() throws Exception { Configuration conf = getConf(); final short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); // All datanodes are on the same rack String racks[] = {"/rack1", "/rack1", "/rack1"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); try { // Create a file with one block with a replication factor of 3 final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); Block b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 1); // Add a new datanode on a different rack String newRacks[] = {"/rack2"}; cluster.startDataNodes(conf, 1, true, null, newRacks); cluster.waitActive(); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
/** * Verify that the NameNode stays in safemode when dfs.safemode.datanode.min * is set to a number greater than the number of live datanodes. */ @Test public void testDatanodeThreshold() throws IOException { cluster.shutdown(); Configuration conf = cluster.getConfiguration(0); conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1); cluster.restartNameNode(); fs = cluster.getFileSystem(); String tipMsg = cluster.getNamesystem().getSafemode(); assertTrue("Safemode tip message doesn't look right: " + tipMsg, tipMsg.contains("The number of live datanodes 0 needs an additional " + "1 live datanodes to reach the minimum number 1." + NEWLINE + "Safe mode will be turned off automatically")); // Start a datanode cluster.startDataNodes(conf, 1, true, null, null); // Wait long enough for safemode check to refire try { Thread.sleep(1000); } catch (InterruptedException ignored) {} // We now should be out of safe mode. assertEquals("", cluster.getNamesystem().getSafemode()); }
@Test public void testSufficientlyReplBlocksUsesNewRack() throws Exception { Configuration conf = getConf(); final short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); // All datanodes are on the same rack String racks[] = {"/rack1", "/rack1", "/rack1"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); try { // Create a file with one block with a replication factor of 3 final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0); // Add a new datanode on a different rack String newRacks[] = {"/rack2"}; cluster.startDataNodes(conf, 1, true, null, newRacks); cluster.waitActive(); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@Test public void testLogRollOnDatanodeDeath() throws IOException, InterruptedException { dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), 3, true, null, null); tableName = getName(); Table table = createTestTable(tableName); TEST_UTIL.waitUntilAllRegionsAssigned(table.getName()); doPut(table, 1); server = TEST_UTIL.getRSForFirstRegionInTable(table.getName()); RegionInfo hri = server.getRegions(table.getName()).get(0).getRegionInfo(); AsyncFSWAL wal = (AsyncFSWAL) server.getWAL(hri); int numRolledLogFiles = AsyncFSWALProvider.getNumRolledLogFiles(wal); DatanodeInfo[] dnInfos = wal.getPipeline(); DataNodeProperties dnProp = TEST_UTIL.getDFSCluster().stopDataNode(dnInfos[0].getName()); TEST_UTIL.getDFSCluster().restartDataNode(dnProp); doPut(table, 2); assertEquals(numRolledLogFiles + 1, AsyncFSWALProvider.getNumRolledLogFiles(wal)); } }
@Test(timeout = 20000) public void testClusterIdMismatchAtStartupWithHA() throws Exception { MiniDFSNNTopology top = new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf("ns1") .addNN(new MiniDFSNNTopology.NNConf("nn0")) .addNN(new MiniDFSNNTopology.NNConf("nn1"))) .addNameservice(new MiniDFSNNTopology.NSConf("ns2") .addNN(new MiniDFSNNTopology.NNConf("nn2").setClusterId("bad-cid")) .addNN(new MiniDFSNNTopology.NNConf("nn3").setClusterId("bad-cid"))); top.setFederation(true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(top) .numDataNodes(0).build(); try { cluster.startDataNodes(conf, 1, true, null, null); // let the initialization be complete Thread.sleep(10000); DataNode dn = cluster.getDataNodes().get(0); assertTrue("Datanode should be running", dn.isDatanodeUp()); assertEquals("Only one BPOfferService should be running", 1, dn.getAllBpOs().length); } finally { cluster.shutdown(); } }