/** * Returns true if the storage directory on the given directory is already * loaded. * @param root the root directory of a {@link StorageDirectory} * @throws IOException if failed to get canonical path. */ protected boolean containsStorageDir(File root) throws IOException { for (StorageDirectory sd : storageDirs) { if (sd.getRoot().getCanonicalPath().equals(root.getCanonicalPath())) { return true; } } return false; }
public void updateNameDirSize() { Map<String, Long> nnDirSizeMap = new HashMap<>(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); if (!sd.isShared()) { nnDirSizeMap.put(sd.getRoot().getAbsolutePath(), sd.getDirecorySize()); } } nameDirSizeMap.clear(); nameDirSizeMap.putAll(nnDirSizeMap); }
/** * Remove block pool level storage directory. * @param absPathToRemove the absolute path of the root for the block pool * level storage to remove. */ void remove(File absPathToRemove) { Preconditions.checkArgument(absPathToRemove.isAbsolute()); LOG.info("Removing block level storage: {}", absPathToRemove); for (Iterator<StorageDirectory> it = getStorageDirs().iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); if (sd.getRoot().getAbsoluteFile().equals(absPathToRemove)) { getStorageDirs().remove(sd); break; } } }
@Override // Storage protected void setFieldsFromProperties( Properties props, StorageDirectory sd) throws IOException { super.setFieldsFromProperties(props, sd); if (layoutVersion == 0) { throw new IOException("NameNode directory " + sd.getRoot() + " is not formatted."); } // Set Block pool ID in version with federation support if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.FEDERATION, getLayoutVersion())) { String sbpid = props.getProperty("blockpoolID"); setBlockPoolID(sd.getRoot(), sbpid); } setDeprecatedPropertiesForUpgrade(props); }
/** Validate and set clusterId from {@link Properties}*/ protected void setClusterId(Properties props, int layoutVersion, StorageDirectory sd) throws InconsistentFSStateException { // Set cluster ID in version that supports federation if (LayoutVersion.supports(getServiceLayoutFeatureMap(), Feature.FEDERATION, layoutVersion)) { String cid = getProperty(props, sd, "clusterID"); if (!(clusterID.equals("") || cid.equals("") || clusterID.equals(cid))) { throw new InconsistentFSStateException(sd.getRoot(), "cluster Id is incompatible with others."); } clusterID = cid; } }
public boolean matchesStorageDirectory(StorageDirectory sd, String bpid) throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED && storageType == StorageType.PROVIDED) { return matchesStorageDirectory(sd); } if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED || storageType == StorageType.PROVIDED) { // only one PROVIDED storage directory can exist; so this cannot match! return false; } // both storage directories are local return this.getBpURI(bpid, Storage.STORAGE_DIR_CURRENT).normalize() .equals(sd.getRoot().toURI().normalize()); }
private File getTrashRootDir(StorageDirectory sd) { return new File(sd.getRoot(), TRASH_ROOT_DIR); }
/** * @return The first image file with the given txid and image type. */ public File getFsImageName(long txid, NameNodeFile nnf) { for (Iterator<StorageDirectory> it = dirIterator(NameNodeDirType.IMAGE); it.hasNext();) { StorageDirectory sd = it.next(); File fsImage = getStorageFile(sd, nnf, txid); if (FileUtil.canRead(sd.getRoot()) && fsImage.exists()) { return fsImage; } } return null; }
File getRoot() { return sd.getRoot(); }
private void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo, final File prevDir, final File tmpDir, final File bbwDir, final File toDir, final int oldLV, Configuration conf) throws IOException { // 3.2. Link block files from <SD>/previous.tmp to <SD>/current linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf); // 4. Write version file under <SD>/current clusterID = nsInfo.getClusterID(); upgradeProperties(sd, conf); // 5. Rename <SD>/previous.tmp to <SD>/previous rename(tmpDir, prevDir); LOG.info("Upgrade of {} is complete", sd.getRoot()); }
@Override public void run() { try { saveFSImage(context, sd, nnf); } catch (SaveNamespaceCancelledException snce) { LOG.info("Cancelled image saving for " + sd.getRoot() + ": " + snce.getMessage()); // don't report an error on the storage dir! } catch (Throwable t) { LOG.error("Unable to save image for " + sd.getRoot(), t); context.reportErrorOnStorageDirectory(sd); } }
@Override public String toString() { return String.format("FileJournalManager(root=%s)", sd.getRoot()); }
@Override public String toString() { return "FSImageSaver for " + sd.getRoot() + " of type " + sd.getStorageDirType(); } }
void upgradeProperties(StorageDirectory sd, Configuration conf) throws IOException { createStorageID(sd, layoutVersion, conf); LOG.info("Updating layout version from {} to {} for storage {}", layoutVersion, HdfsServerConstants.DATANODE_LAYOUT_VERSION, sd.getRoot()); layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; writeProperties(sd); }
/** * Directory {@code edits.sync} temporarily holds the log segments * downloaded through {@link JournalNodeSyncer} before they are moved to * {@code current} directory. * * @return the directory path */ File getEditsSyncDir() { return new File(sd.getRoot(), STORAGE_EDITS_SYNC); }
@Override protected void setFieldsFromProperties(Properties props, StorageDirectory sd) throws IOException { setLayoutVersion(props, sd); setNamespaceID(props, sd); setcTime(props, sd); String sbpid = props.getProperty("blockpoolID"); setBlockPoolID(sd.getRoot(), sbpid); }
/** Check if upgrade is in progress. */ public static void checkUpgrade(NNStorage storage) throws IOException { // Upgrade or rolling upgrade is allowed only if there are // no previous fs states in any of the local directories for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext();) { StorageDirectory sd = it.next(); if (sd.getPreviousDir().exists()) throw new InconsistentFSStateException(sd.getRoot(), "previous fs state should not exist during upgrade. " + "Finalize or rollback first."); } }
/** Create new dfs name directory. Caution: this destroys all files * in this filesystem. */ private void format(StorageDirectory sd) throws IOException { sd.clearDirectory(); // create currrent dir writeProperties(sd); writeTransactionIdFile(sd, 0); LOG.info("Storage directory {} has been successfully formatted.", sd.getRoot()); }
@Override public void doPreUpgrade() throws IOException { LOG.info("Starting upgrade of edits directory " + sd.getRoot()); try { NNUpgradeUtil.doPreUpgrade(conf, sd); } catch (IOException ioe) { LOG.error("Failed to move aside pre-upgrade storage " + "in image directory " + sd.getRoot(), ioe); throw ioe; } }
/** * generate storage list (debug line) */ public String listStorageDirectories() { StringBuilder buf = new StringBuilder(); for (StorageDirectory sd : storageDirs) { buf.append(sd.getRoot() + "(" + sd.getStorageDirType() + ");"); } return buf.toString(); }