/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
/** * Clone the supplied configuration but remove the shared edits dirs. * * @param conf Supplies the original configuration. * @return Cloned configuration without the shared edit dirs. * @throws IOException on failure to generate the configuration. */ private static Configuration getConfigurationWithoutSharedEdits( Configuration conf) throws IOException { List<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf, false); String editsDirsString = Joiner.on(",").join(editsDirs); Configuration confWithoutShared = new Configuration(conf); confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); confWithoutShared.setStrings(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsDirsString); return confWithoutShared; }
public static List<URI> getNamespaceEditsDirs(Configuration conf, boolean includeShared) throws IOException { // Use a LinkedHashSet so that order is maintained while we de-dup // the entries. LinkedHashSet<URI> editsDirs = new LinkedHashSet<URI>(); if (includeShared) { List<URI> sharedDirs = getSharedEditsDirs(conf); // Fail until multiple shared edits directories are supported (HDFS-2782) if (sharedDirs.size() > 1) { throw new IOException( "Multiple shared edits directories are not yet supported"); } // First add the shared edits dirs. It's critical that the shared dirs // are added first, since JournalSet syncs them in the order they are listed, // and we need to make sure all edits are in place in the shared storage // before they are replicated locally. See HDFS-2874. for (URI dir : sharedDirs) { if (!editsDirs.add(dir)) { LOG.warn("Edits URI " + dir + " listed multiple times in " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY + ". Ignoring duplicates."); } } } // Now add the non-shared dirs. for (URI dir : getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY)) { if (!editsDirs.add(dir)) {
FSNamesystem.getNamespaceDirs(conf); final Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf); final Collection<URI> requiredEditsDirs = FSNamesystem.getRequiredNamespaceEditsDirs(conf);
FSNamesystem.getNamespaceEditsDirs(conf), new Predicate<URI>() { @Override
editUrisToFormat = FSNamesystem.getNamespaceEditsDirs( conf, false); sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf);
/** * Get the directories where the namenode stores its edits. */ public Collection<URI> getNameEditsDirs() { return FSNamesystem.getNamespaceEditsDirs(conf); }
dirsToPrompt.addAll(sharedDirs); List<URI> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf);
FSImage fsImage = new FSImage(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); FSNamesystem namesystem = new FSNamesystem(conf, fsImage, false); StartupOption startOpt = NameNode.getStartupOption(conf);
/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
/** * Get the directories where the namenode stores its edits. */ public Collection<URI> getNameEditsDirs(int nnIndex) throws IOException { return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf); }
/** * Clone the supplied configuration but remove the shared edits dirs. * * @param conf Supplies the original configuration. * @return Cloned configuration without the shared edit dirs. * @throws IOException on failure to generate the configuration. */ private static Configuration getConfigurationWithoutSharedEdits( Configuration conf) throws IOException { List<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf, false); String editsDirsString = Joiner.on(",").join(editsDirs); Configuration confWithoutShared = new Configuration(conf); confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); confWithoutShared.setStrings(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsDirsString); return confWithoutShared; }
/** * Clone the supplied configuration but remove the shared edits dirs. * * @param conf Supplies the original configuration. * @return Cloned configuration without the shared edit dirs. * @throws IOException on failure to generate the configuration. */ private static Configuration getConfigurationWithoutSharedEdits( Configuration conf) throws IOException { List<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf, false); String editsDirsString = Joiner.on(",").join(editsDirs); Configuration confWithoutShared = new Configuration(conf); confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); confWithoutShared.setStrings(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsDirsString); return confWithoutShared; }
/** * Tests that the namenode edits dirs are gotten with duplicates removed */ @Test public void testUniqueEditDirs() throws IOException { Configuration config = new Configuration(); config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, " + "file://edits/dir1,file://edits/dir1"); // overlapping internally // getNamespaceEditsDirs removes duplicates Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config); assertEquals(2, editsDirs.size()); }
/** * Tests that the namenode edits dirs and shared edits dirs are gotten with * duplicates removed */ @Test public void testHAUniqueEditDirs() throws IOException { Configuration conf = new Configuration(); conf.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, " + "file://edits/shared/dir"); // overlapping conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir"); // getNamespaceEditsDirs removes duplicates across edits and shared.edits Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf); assertEquals(2, editsDirs.size()); }
private void corruptNameNodeFiles() throws IOException { // now corrupt/delete the directrory List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config); List<URI> nameEditsDirs = FSNamesystem.getNamespaceEditsDirs(config); // get name dir and its length, then delete and recreate the directory File dir = new File(nameDirs.get(0).getPath()); // has only one this.fsimageLength = new File(new File(dir, Storage.STORAGE_DIR_CURRENT), NameNodeFile.IMAGE.getName()).length(); if(dir.exists() && !(FileUtil.fullyDelete(dir))) throw new IOException("Cannot remove directory: " + dir); LOG.info("--removed dir "+dir + ";len was ="+ this.fsimageLength); if (!dir.mkdirs()) throw new IOException("Cannot create directory " + dir); dir = new File( nameEditsDirs.get(0).getPath()); //has only one this.editsLength = new File(new File(dir, Storage.STORAGE_DIR_CURRENT), NameNodeFile.EDITS.getName()).length(); if(dir.exists() && !(FileUtil.fullyDelete(dir))) throw new IOException("Cannot remove directory: " + dir); if (!dir.mkdirs()) throw new IOException("Cannot create directory " + dir); LOG.info("--removed dir and recreated "+dir + ";len was ="+ this.editsLength); }
/** * Multiple shared edits directories is an invalid configuration. */ @Test public void testMultipleSharedDirsFails() throws Exception { Configuration conf = new Configuration(); URI sharedA = new URI("file:///shared-A"); URI sharedB = new URI("file:///shared-B"); URI localA = new URI("file:///local-A"); conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, Joiner.on(",").join(sharedA,sharedB)); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, localA.toString()); try { FSNamesystem.getNamespaceEditsDirs(conf); fail("Allowed multiple shared edits directories"); } catch (IOException ioe) { assertEquals("Multiple shared edits directories are not yet supported", ioe.getMessage()); } }
/** * Construct FSEditLog with default configuration, taking editDirs from NNStorage * * @param storage Storage object used by namenode */ private static FSEditLog getFSEditLog(NNStorage storage) throws IOException { Configuration conf = new Configuration(); // Make sure the edits dirs are set in the provided configuration object. conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, StringUtils.join(",", storage.getEditsDirectories())); FSEditLog log = new FSEditLog(conf, storage, FSNamesystem.getNamespaceEditsDirs(conf)); return log; }
private String getClusterId(Configuration config) throws IOException { // see if cluster id not empty. Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(config); List<URI> editsToFormat = FSNamesystem.getNamespaceEditsDirs(config); FSImage fsImage = new FSImage(config, dirsToFormat, editsToFormat); Iterator<StorageDirectory> sdit = fsImage.getStorage().dirIterator(NNStorage.NameNodeDirType.IMAGE); StorageDirectory sd = sdit.next(); Properties props = Storage.readPropertiesFile(sd.getVersionFile()); String cid = props.getProperty("clusterID"); LOG.info("successfully formated : sd="+sd.getCurrentDir() + ";cid="+cid); return cid; }