@Before public void setup() throws Exception { Configuration hdfsConf = new HdfsConfiguration(); hdfsConf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); String namenodeDir = new File(MiniDFSCluster.getBaseDirectory(), "name"). getAbsolutePath(); hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir); hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir); hdfsConf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); cluster = new MiniDFSCluster.Builder(hdfsConf).numDataNodes(15).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy(ecPolicy.getName()); fs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName()); cluster.waitActive(); conf = new Configuration(); submitDir = new Path("/"); testFile = new Path("/testfile"); DFSTestUtil.writeFile(fs, testFile, StripedFileTestUtil.generateBytes(BLOCKSIZE)); conf.set(FileInputFormat.INPUT_DIR, fs.getUri().toString() + testFile.toString()); }
private static void waitForBlockDeleted(String blockName, int dnIndex) throws IOException, InterruptedException { File baseDir = new File(MiniDFSCluster.getBaseDirectory(), "data"); File blockFile1 = new File(baseDir, "data" + (2*dnIndex+1) + MiniDFSCluster.FINALIZED_DIR_NAME + blockName); File blockFile2 = new File(baseDir, "data" + (2*dnIndex+2) + MiniDFSCluster.FINALIZED_DIR_NAME + blockName); while (blockFile1.exists() || blockFile2.exists()) { Thread.sleep(100); } } }
private File[] getDataNodeDirs(int i) throws IOException { String base_dir = MiniDFSCluster.getBaseDirectory(); File data_dir = new File(base_dir, "data"); File dir1 = new File(data_dir, "data"+(2*i+1)); File dir2 = new File(data_dir, "data"+(2*i+2)); if (dir1.isDirectory() && dir2.isDirectory()) { File[] dir = new File[2]; dir[0] = new File(dir1, MiniDFSCluster.FINALIZED_DIR_NAME); dir[1] = new File(dir2, MiniDFSCluster.FINALIZED_DIR_NAME); return dir; } return new File[0]; }
public void formatDataNodeDirs() throws IOException { base_dir = new File(getBaseDirectory()); data_dir = new File(base_dir, "data"); if (data_dir.exists() && !FileUtil.fullyDelete(data_dir)) { throw new IOException("Cannot remove data directory: " + data_dir); } }
/** * Change the length of a block at datanode dnIndex */ static boolean changeReplicaLength(String blockName, int dnIndex, int lenDelta) throws IOException { File baseDir = new File(MiniDFSCluster.getBaseDirectory(), "data"); for (int i=dnIndex*2; i<dnIndex*2+2; i++) { File blockFile = new File(baseDir, "data" + (i+1) + MiniDFSCluster.FINALIZED_DIR_NAME + blockName); if (blockFile.exists()) { RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw"); raFile.setLength(raFile.length()+lenDelta); raFile.close(); return true; } } return false; }
private Configuration getConf() throws IOException { String baseDir = MiniDFSCluster.getBaseDirectory(); String nameDirs = fileAsURI(new File(baseDir, "name1")) + "," + fileAsURI(new File(baseDir, "name2")); Configuration conf = new HdfsConfiguration(); FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDirs); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameDirs); conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false); return conf; } }
SecondaryNameNode startSecondaryNameNode(Configuration conf, int index) throws IOException { Configuration snnConf = new Configuration(conf); snnConf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); snnConf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY, MiniDFSCluster.getBaseDirectory() + "/2nn-" + index); return new SecondaryNameNode(snnConf); }
private void initConfig() { conf.set(DFS_NAMESERVICE_ID, "ns1"); conf.set(DFS_HA_NAMENODES_KEY_PREFIX + ".ns1", "nn1"); conf.set(DFS_HA_NAMENODE_ID_KEY, "nn1"); conf.set(DFS_NAMENODE_NAME_DIR_KEY + ".ns1.nn1", MiniDFSCluster.getBaseDirectory() + "1"); conf.unset(DFS_NAMENODE_NAME_DIR_KEY); }
/** * Get a storage directory for a datanode. * <ol> * <li><base directory>/data/data<2*dnIndex + 1></li> * <li><base directory>/data/data<2*dnIndex + 2></li> * </ol> * * @param dnIndex datanode index (starts from 0) * @param dirIndex directory index. * @return Storage directory */ public File getStorageDir(int dnIndex, int dirIndex) { return new File(getBaseDirectory(), getStorageDirPath(dnIndex, dirIndex)); }
@Before public void cleanupCluster() throws IOException { File hdfsDir = new File(MiniDFSCluster.getBaseDirectory()).getCanonicalFile(); System.out.println("cleanupCluster deleting " + hdfsDir); if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'"); } }
@Before public void setUp() { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); faultInjector = Mockito.mock(CheckpointFaultInjector.class); CheckpointFaultInjector.instance = faultInjector; }
@Before public void setUp() throws IOException { conf = new Configuration(); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, fileAsURI(new File(MiniDFSCluster.getBaseDirectory(), "namenode")).toString()); NameNode.initMetrics(conf, NamenodeRole.NAMENODE); fs = null; fsIsReady = true; }
@After public void cleanUp() { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); }
@After public void cleanUp() { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); }
@After public void cleanUp() { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); }
@Before public void setUp() { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); }
private Configuration getConf() throws IOException { String baseDir = MiniDFSCluster.getBaseDirectory(); String nameDirs = fileAsURI(new File(baseDir, "name1")) + "," + fileAsURI(new File(baseDir, "name2")); Configuration conf = new HdfsConfiguration(); FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDirs); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameDirs); conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false); return conf; } }
/** * This test runs all benchmarks defined in {@link NNThroughputBenchmark}. */ @Test public void testNNThroughput() throws Exception { Configuration conf = new HdfsConfiguration(); File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath()); DFSTestUtil.formatNameNode(conf); NNThroughputBenchmark.runBenchmark(conf, new String[] {"-op", "all"}); }
/** * This test runs all benchmarks defined in {@link NNThroughputBenchmark}, * with explicit local -fs option. */ @Test(timeout = 120000) public void testNNThroughputWithFsOption() throws Exception { Configuration conf = new HdfsConfiguration(); File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath()); DFSTestUtil.formatNameNode(conf); NNThroughputBenchmark.runBenchmark(conf, new String[] {"-fs", "file:///", "-op", "all"}); }
/** * Test that FSNamesystem#clear clears all leases. */ @Test public void testFSNamespaceClearLeases() throws Exception { Configuration conf = new HdfsConfiguration(); File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name"); conf.set(DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath()); NameNode.initMetrics(conf, NamenodeRole.NAMENODE); DFSTestUtil.formatNameNode(conf); FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); LeaseManager leaseMan = fsn.getLeaseManager(); leaseMan.addLease("client1", "importantFile"); assertEquals(1, leaseMan.countLease()); fsn.clear(); leaseMan = fsn.getLeaseManager(); assertEquals(0, leaseMan.countLease()); }