/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
return Lists.newArrayList(getNamespaceDirs(conf)); } else { return Lists.newArrayList(editsDirs);
FSNamesystem.getNamespaceDirs(conf); final Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf);
dirsToFormat = FSNamesystem.getNamespaceDirs(conf); editUrisToFormat = FSNamesystem.getNamespaceEditsDirs( conf, false);
/** * Get the directories where the namenode stores its image. */ public Collection<URI> getNameDirs() { return FSNamesystem.getNamespaceDirs(conf); }
Collection<URI> nameDirsToFormat = FSNamesystem.getNamespaceDirs(conf); List<URI> sharedDirs = FSNamesystem.getSharedEditsDirs(conf); List<URI> dirsToPrompt = new ArrayList<URI>();
FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); FSNamesystem namesystem = new FSNamesystem(conf, fsImage, false);
/** * Get the directories where the namenode stores its image. */ public Collection<URI> getNameDirs(int nnIndex) { return FSNamesystem.getNamespaceDirs(nameNodes[nnIndex].conf); }
/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
@Override protected void doExecute(Configuration conf) throws Exception { Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); for (Iterator<File> it = dirsToFormat.iterator(); it.hasNext(); ) { File curDir = it.next(); if (!curDir.exists()) { continue; } if (!force) { System.err.print("Re-format filesystem in " + curDir + " ? (Y or N) "); System.err.flush(); if (!(System.in.read() == 'Y')) { System.err.println("Format aborted in " + curDir); return; } while (System.in.read() != '\n') ; // discard the enter-key } } NameNode.format(conf); }
@Override protected NameNode doCreate(Dictionary properties) throws Exception { Configuration conf = new Configuration(); for (Enumeration e = properties.keys(); e.hasMoreElements();) { Object key = e.nextElement(); Object val = properties.get(key); conf.set( key.toString(), val.toString() ); } boolean exists = false; for (File file : FSNamesystem.getNamespaceDirs(conf)) { exists |= file.exists(); } if (!exists) { NameNode.format(conf); } NameNode nameNode = NameNode.createNameNode(null, conf); return nameNode; }
private void corruptNameNodeFiles() throws IOException { // now corrupt/delete the directrory List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config); List<URI> nameEditsDirs = (List<URI>)FSNamesystem.getNamespaceEditsDirs(config); // get name dir and its length, then delete and recreate the directory File dir = new File(nameDirs.get(0).getPath()); // has only one this.fsimageLength = new File(new File(dir, "current"), NameNodeFile.IMAGE.getName()).length(); if(dir.exists() && !(FileUtil.fullyDelete(dir))) throw new IOException("Cannot remove directory: " + dir); LOG.info("--removed dir "+dir + ";len was ="+ this.fsimageLength); if (!dir.mkdirs()) throw new IOException("Cannot create directory " + dir); dir = new File( nameEditsDirs.get(0).getPath()); //has only one this.editsLength = new File(new File(dir, "current"), NameNodeFile.EDITS.getName()).length(); if(dir.exists() && !(FileUtil.fullyDelete(dir))) throw new IOException("Cannot remove directory: " + dir); if (!dir.mkdirs()) throw new IOException("Cannot create directory " + dir); LOG.info("--removed dir and recreated "+dir + ";len was ="+ this.editsLength); }
private static boolean finalize(Configuration conf, boolean isConfirmationNeeded ) throws IOException { Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); Collection<File> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat, editDirsToFormat), conf); System.err.print( "\"finalize\" will remove the previous state of the files system.\n" + "Recent upgrade will become permanent.\n" + "Rollback option will not be available anymore.\n"); if (isConfirmationNeeded) { System.err.print("Finalize filesystem state ? (Y or N) "); if (!(System.in.read() == 'Y')) { System.err.println("Finalize aborted."); return true; } while(System.in.read() != '\n'); // discard the enter-key } nsys.dir.fsImage.finalizeUpgrade(); return false; }
private static boolean finalize(Configuration conf, boolean isConfirmationNeeded ) throws IOException { Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); Collection<File> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat, editDirsToFormat), conf); System.err.print( "\"finalize\" will remove the previous state of the files system.\n" + "Recent upgrade will become permanent.\n" + "Rollback option will not be available anymore.\n"); if (isConfirmationNeeded) { System.err.print("Finalize filesystem state ? (Y or N) "); if (!(System.in.read() == 'Y')) { System.err.println("Finalize aborted."); return true; } while(System.in.read() != '\n'); // discard the enter-key } nsys.dir.fsImage.finalizeUpgrade(); return false; }
private void corruptNameNodeFiles() throws IOException { // now corrupt/delete the directrory List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config); List<URI> nameEditsDirs = FSNamesystem.getNamespaceEditsDirs(config); // get name dir and its length, then delete and recreate the directory File dir = new File(nameDirs.get(0).getPath()); // has only one this.fsimageLength = new File(new File(dir, Storage.STORAGE_DIR_CURRENT), NameNodeFile.IMAGE.getName()).length(); if(dir.exists() && !(FileUtil.fullyDelete(dir))) throw new IOException("Cannot remove directory: " + dir); LOG.info("--removed dir "+dir + ";len was ="+ this.fsimageLength); if (!dir.mkdirs()) throw new IOException("Cannot create directory " + dir); dir = new File( nameEditsDirs.get(0).getPath()); //has only one this.editsLength = new File(new File(dir, Storage.STORAGE_DIR_CURRENT), NameNodeFile.EDITS.getName()).length(); if(dir.exists() && !(FileUtil.fullyDelete(dir))) throw new IOException("Cannot remove directory: " + dir); if (!dir.mkdirs()) throw new IOException("Cannot create directory " + dir); LOG.info("--removed dir and recreated "+dir + ";len was ="+ this.editsLength); }
private void parseConfAndFindOtherNN() throws IOException { Configuration conf = getConf(); nsId = DFSUtil.getNamenodeNameServiceId(conf); if (!HAUtil.isHAEnabled(conf, nsId)) { throw new HadoopIllegalArgumentException( "HA is not enabled for this namenode."); } nnId = HAUtil.getNameNodeId(conf, nsId); NameNode.initializeGenericKeys(conf, nsId, nnId); if (!HAUtil.usesSharedEditsDir(conf)) { throw new HadoopIllegalArgumentException( "Shared edits storage is not enabled for this namenode."); } Configuration otherNode = HAUtil.getConfForOtherNode(conf); otherNNId = HAUtil.getNameNodeId(otherNode, nsId); otherIpcAddr = NameNode.getServiceAddress(otherNode, true); Preconditions.checkArgument(otherIpcAddr.getPort() != 0 && !otherIpcAddr.getAddress().isAnyLocalAddress(), "Could not determine valid IPC address for other NameNode (%s)" + ", got: %s", otherNNId, otherIpcAddr); final String scheme = DFSUtil.getHttpClientScheme(conf); otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost( otherIpcAddr.getHostName(), otherNode, scheme).toURL(); dirsToFormat = FSNamesystem.getNamespaceDirs(conf); editUrisToFormat = FSNamesystem.getNamespaceEditsDirs( conf, false); sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf); }
/** * Corrupts the MD5 sum of the fsimage. * * @param corruptAll * whether to corrupt one or all of the MD5 sums in the configured * namedirs * @throws IOException */ private void corruptFSImageMD5(boolean corruptAll) throws IOException { List<URI> nameDirs = (List<URI>)FSNamesystem.getNamespaceDirs(config); // Corrupt the md5 files in all the namedirs for (URI uri: nameDirs) { // Directory layout looks like: // test/data/dfs/nameN/current/{fsimage,edits,...} File nameDir = new File(uri.getPath()); File dfsDir = nameDir.getParentFile(); assertEquals(dfsDir.getName(), "dfs"); // make sure we got right dir // Set the md5 file to all zeros File imageFile = new File(nameDir, Storage.STORAGE_DIR_CURRENT + "/" + NNStorage.getImageFileName(0)); MD5FileUtils.saveMD5File(imageFile, new MD5Hash(new byte[16])); // Only need to corrupt one if !corruptAll if (!corruptAll) { break; } } }
private void parseConfAndFindOtherNN() throws IOException { Configuration conf = getConf(); nsId = DFSUtil.getNamenodeNameServiceId(conf); if (!HAUtil.isHAEnabled(conf, nsId)) { throw new HadoopIllegalArgumentException( "HA is not enabled for this namenode."); } nnId = HAUtil.getNameNodeId(conf, nsId); NameNode.initializeGenericKeys(conf, nsId, nnId); if (!HAUtil.usesSharedEditsDir(conf)) { throw new HadoopIllegalArgumentException( "Shared edits storage is not enabled for this namenode."); } Configuration otherNode = HAUtil.getConfForOtherNode(conf); otherNNId = HAUtil.getNameNodeId(otherNode, nsId); otherIpcAddr = NameNode.getServiceAddress(otherNode, true); Preconditions.checkArgument(otherIpcAddr.getPort() != 0 && !otherIpcAddr.getAddress().isAnyLocalAddress(), "Could not determine valid IPC address for other NameNode (%s)" + ", got: %s", otherNNId, otherIpcAddr); final String scheme = DFSUtil.getHttpClientScheme(conf); otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost( otherIpcAddr.getHostName(), otherNode, scheme).toURL(); dirsToFormat = FSNamesystem.getNamespaceDirs(conf); editUrisToFormat = FSNamesystem.getNamespaceEditsDirs( conf, false); sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf); }
private String getClusterId(Configuration config) throws IOException { // see if cluster id not empty. Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(config); List<URI> editsToFormat = FSNamesystem.getNamespaceEditsDirs(config); FSImage fsImage = new FSImage(config, dirsToFormat, editsToFormat); Iterator<StorageDirectory> sdit = fsImage.getStorage().dirIterator(NNStorage.NameNodeDirType.IMAGE); StorageDirectory sd = sdit.next(); Properties props = Storage.readPropertiesFile(sd.getVersionFile()); String cid = props.getProperty("clusterID"); LOG.info("successfully formated : sd="+sd.getCurrentDir() + ";cid="+cid); return cid; }