public NamespaceInfo getNamespaceInfo() { return new NamespaceInfo( getNamespaceID(), getClusterID(), null, getCTime()); }
assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException("Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); if (!blockpoolID.equals(nsInfo.getBlockPoolID())) { throw new IOException("Incompatible blockpoolIDs in " + sd.getRoot().getCanonicalPath() + ": namenode blockpoolID = " + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); && this.cTime == nsInfo.getCTime()) { return false; // regular startup || this.cTime < nsInfo.getCTime()) { + " CTime = " + this.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime());
public static NamespaceInfoProto convert(NamespaceInfo info) { NamespaceInfoProto.Builder builder = NamespaceInfoProto.newBuilder(); builder.setBlockPoolID(info.getBlockPoolID()) .setBuildVersion(info.getBuildVersion()) .setUnused(0) .setStorageInfo(PBHelper.convert((StorageInfo)info)) .setSoftwareVersion(info.getSoftwareVersion()) .setCapabilities(info.getCapabilities()); HAServiceState state = info.getState(); if(state != null) { builder.setState(convert(info.getState())); } return builder.build(); }
/** * Get the BlockPoolSliceStorage from {@link bpStorageMap}. * If the object is not found, create a new object and put it to the map. */ synchronized BlockPoolSliceStorage getBlockPoolSliceStorage( final NamespaceInfo nsInfo) { final String bpid = nsInfo.getBlockPoolID(); BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpid); if (bpStorage == null) { bpStorage = new BlockPoolSliceStorage(nsInfo.getNamespaceID(), bpid, nsInfo.getCTime(), nsInfo.getClusterID()); bpStorageMap.put(bpid, bpStorage); } return bpStorage; }
void checkConsistentNamespace(NamespaceInfo nsInfo) throws IOException { if (nsInfo.getNamespaceID() != getNamespaceID()) { throw new IOException("Incompatible namespaceID for journal " + this.sd + ": NameNode has nsId " + nsInfo.getNamespaceID() + " but storage has nsId " + getNamespaceID()); } if (!nsInfo.getClusterID().equals(getClusterID())) { throw new IOException("Incompatible clusterID for journal " + this.sd + ": NameNode has clusterId '" + nsInfo.getClusterID() + "' but storage has clusterId '" + getClusterID() + "'"); } }
/** * Format all available storage directories. */ public void format(NamespaceInfo nsInfo) throws IOException { Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 || nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION, "Bad layout version: %s", nsInfo.getLayoutVersion()); this.setStorageInfo(nsInfo); this.blockpoolID = nsInfo.getBlockPoolID(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); format(sd); } }
public void write(DataOutput out) throws IOException { UTF8.writeString(out, getBuildVersion()); out.writeInt(getLayoutVersion()); out.writeInt(getNamespaceID()); out.writeLong(getCTime()); out.writeInt(getDistributedUpgradeVersion()); }
LOG.error("Layout version on remote node (" + nsInfo.getLayoutVersion() + ") does not match " + "this node's layout version (" + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + ")"); " Other NN's HTTP address: " + proxyInfo.getHttpAddress() + "\n" + " Other NN's IPC address: " + proxyInfo.getIpcAddress() + "\n" + " Namespace ID: " + nsInfo.getNamespaceID() + "\n" + " Block pool ID: " + nsInfo.getBlockPoolID() + "\n" + " Cluster ID: " + nsInfo.getClusterID() + "\n" + " Layout version: " + nsInfo.getLayoutVersion() + "\n" + " isUpgradeFinalized: " + isUpgradeFinalized + "\n" + "=====================================================");
/** * Format a block pool slice storage. * @param bpSdir the block pool storage * @param nsInfo the name space info * @throws IOException Signals that an I/O exception has occurred. */ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException { LOG.info("Formatting block pool {} directory {}", blockpoolID, bpSdir .getCurrentDir()); bpSdir.clearDirectory(); // create directory this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); writeProperties(bpSdir); }
private NamespaceInfo handshake() throws IOException { NamespaceInfo nsInfo = new NamespaceInfo(); while (shouldRun && shouldServiceRun) { try { if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion() )) { errorMsg = "Incompatible build versions: namenode BV = " + nsInfo.getBuildVersion() + "; datanode BV = " + Storage.getBuildVersion(); LOG.warn( errorMsg ); assert FSConstants.LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Data-node and name-node layout versions must be the same." + "Expected: "+ FSConstants.LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion(); return nsInfo;
prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException(bpSd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = " + nsInfo.getCTime()); + "CTime = {}", bpSd.getRoot(), nsInfo.getLayoutVersion(), nsInfo.getCTime()); File tmpDir = bpSd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist.";
private static NamespaceInfo handshake(NamenodeProtocol namenode) throws IOException, SocketTimeoutException { NamespaceInfo nsInfo; nsInfo = namenode.versionRequest(); // throws SocketTimeoutException String errorMsg = null; // verify build version if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion())) { errorMsg = "Incompatible build versions: active name-node BV = " + nsInfo.getBuildVersion() + "; backup node BV = " + Storage.getBuildVersion(); LOG.error(errorMsg); throw new IOException(errorMsg); } assert HdfsServerConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Active and backup node layout versions must be the same. Expected: " + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion(); return nsInfo; }
@VisibleForTesting NamespaceInfo setNamespaceInfo(NamespaceInfo nsInfo) throws IOException { writeLock(); try { NamespaceInfo old = bpNSInfo; if (bpNSInfo != null && nsInfo != null) { checkNSEquality(bpNSInfo.getBlockPoolID(), nsInfo.getBlockPoolID(), "Blockpool ID"); checkNSEquality(bpNSInfo.getNamespaceID(), nsInfo.getNamespaceID(), "Namespace ID"); checkNSEquality(bpNSInfo.getClusterID(), nsInfo.getClusterID(), "Cluster ID"); } bpNSInfo = nsInfo; // cache the block pool id for lock-free access. bpId = (nsInfo != null) ? nsInfo.getBlockPoolID() : null; return old; } finally { writeUnlock(); } }
@Test public void testConvertNamespaceInfo() { NamespaceInfo info = new NamespaceInfo(37, "clusterID", "bpID", 2300); NamespaceInfoProto proto = PBHelper.convert(info); NamespaceInfo info2 = PBHelper.convert(proto); compare(info, info2); //Compare the StorageInfo assertEquals(info.getBlockPoolID(), info2.getBlockPoolID()); assertEquals(info.getBuildVersion(), info2.getBuildVersion()); }
private void doUpgrade(String name, final StorageDirectory bpSd, NamespaceInfo nsInfo, final File bpPrevDir, final File bpTmpDir, final File bpCurDir, final int oldLV, Configuration conf) throws IOException { // 3. Create new <SD>/current with block files hardlinks and VERSION linkAllBlocks(bpTmpDir, bpCurDir, oldLV, conf); this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; assert this.namespaceID == nsInfo.getNamespaceID() : "Data-node and name-node layout versions must be the same."; this.cTime = nsInfo.getCTime(); writeProperties(bpSd); // 4.rename <SD>/current/<bpid>/previous.tmp to // <SD>/current/<bpid>/previous rename(bpTmpDir, bpPrevDir); LOG.info("Upgrade of {} is complete", name); }
/** * Format the local storage with the given namespace. */ void format(NamespaceInfo nsInfo, boolean force) throws IOException { Preconditions.checkState(nsInfo.getNamespaceID() != 0, "can't format with uninitialized namespace info: %s", nsInfo); LOG.info("Formatting journal id : " + journalId + " with namespace info: " + nsInfo + " and force: " + force); storage.format(nsInfo, force); refreshCachedData(); }
private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException { return (nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION); }
List<StorageDirectory> succeedDirs = Lists.newArrayList(); try { if (containsStorageDir(location, nsInfo.getBlockPoolID())) { throw new IOException( "BlockPoolSliceStorage.recoverTransitionRead: " + } catch (IOException e) { LOG.warn("Failed to analyze storage directories for block pool {}", nsInfo.getBlockPoolID(), e); throw e;
throws IOException { StorageDirectory sd = new StorageDirectory( nsInfo.getBlockPoolID(), null, true, location); try { StorageState curState = sd.analyzeStorage(startOpt, this, true); case NON_EXISTENT: LOG.info("Block pool storage directory for location {} and block pool" + " id {} does not exist", location, nsInfo.getBlockPoolID()); throw new IOException("Storage directory for location " + location + " and block pool id " + nsInfo.getBlockPoolID() + " does not exist"); case NOT_FORMATTED: // format LOG.info("Block pool storage directory for location {} and block pool" + " id {} is not formatted. Formatting ...", location, nsInfo.getBlockPoolID()); format(sd, nsInfo); break; if (getCTime() != nsInfo.getCTime()) { throw new IOException("Datanode CTime (=" + getCTime() + ") is not equal to namenode CTime (=" + nsInfo.getCTime() + ")");
storage.setBlockPoolID(nsInfo.getBlockPoolID()); storage.setClusterID(nsInfo.getClusterID()); } else { nsInfo.validateStorage(storage);