void upgradeProperties(StorageDirectory sd, Configuration conf) throws IOException { createStorageID(sd, layoutVersion, conf); LOG.info("Updating layout version from {} to {} for storage {}", layoutVersion, HdfsServerConstants.DATANODE_LAYOUT_VERSION, sd.getRoot()); layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; writeProperties(sd); }
private static boolean createStorageID(StorageDirectory sd, int lv, Configuration conf) { // Clusters previously upgraded from layout versions earlier than // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a // new storage ID. We check for that and fix it now. final boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, lv) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); return createStorageID(sd, !haveValidStorageId, conf); }
void format(StorageDirectory sd, NamespaceInfo nsInfo, String newDatanodeUuid, Configuration conf) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(newDatanodeUuid); createStorageID(sd, false, conf); writeProperties(sd); }
Configuration conf) throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { createStorageID(sd, layoutVersion, conf); return false; // regular start up for PROVIDED storage directories createStorageID(sd, layoutVersion, conf); return false; // need to write properties
void upgradeProperties(StorageDirectory sd) throws IOException { createStorageID(sd, layoutVersion); LOG.info("Updating layout version from " + layoutVersion + " to " + HdfsConstants.DATANODE_LAYOUT_VERSION + " for storage " + sd.getRoot()); layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; writeProperties(sd); }
/** * Initializes the {@link #data}. The initialization is done only once, when * handshake with the the first namenode is completed. */ private synchronized void initFsDataSet(Configuration conf, AbstractList<File> dataDirs, int numNamespaces) throws IOException { if (data != null) { // Already initialized return; } // get version and id info from the name-node boolean simulatedFSDataset = conf.getBoolean("dfs.datanode.simulateddatastorage", false); if (simulatedFSDataset) { storage.createStorageID(selfAddr.getPort()); // it would have been better to pass storage as a parameter to // constructor below - need to augment ReflectionUtils used below. conf.set("dfs.datanode.StorageId", storage.getStorageID()); try { data = (FSDatasetInterface) ReflectionUtils.newInstance( Class.forName( "org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset"), conf); } catch (ClassNotFoundException e) { throw new IOException(StringUtils.stringifyException(e)); } } else { data = new FSDataset(this, conf, numNamespaces); } }
private static boolean createStorageID(StorageDirectory sd, int lv) { // Clusters previously upgraded from layout versions earlier than // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a // new storage ID. We check for that and fix it now. final boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, lv) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); return createStorageID(sd, !haveValidStorageId); }
void upgradeProperties(StorageDirectory sd) throws IOException { createStorageID(sd, layoutVersion); LOG.info("Updating layout version from " + layoutVersion + " to " + HdfsConstants.DATANODE_LAYOUT_VERSION + " for storage " + sd.getRoot()); layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; writeProperties(sd); }
private static Storage.StorageDirectory createStorageDirectory(File root) { Storage.StorageDirectory sd = new Storage.StorageDirectory(root); DataStorage.createStorageID(sd, false); return sd; }
private static boolean createStorageID(StorageDirectory sd, int lv) { // Clusters previously upgraded from layout versions earlier than // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a // new storage ID. We check for that and fix it now. final boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, lv) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); return createStorageID(sd, !haveValidStorageId); }
void format(StorageDirectory sd, NamespaceInfo nsInfo, String datanodeUuid) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(datanodeUuid); createStorageID(sd, false); writeProperties(sd); }
void format(StorageDirectory sd, NamespaceInfo nsInfo, String datanodeUuid) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(datanodeUuid); createStorageID(sd, false); writeProperties(sd); }
createStorageID(datanode.getPort());
createStorageID(sd, layoutVersion); return false; // need to write properties
createStorageID(sd, layoutVersion); return false; // need to write properties
/** * Create a <code>version</code> file for datanode inside the specified parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent directory where namenode VERSION file is stored * @param version StorageInfo to create VERSION file from * @param bpid Block pool Id * @param bpidToWrite Block pool Id to write into the version file */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version); storage.setDatanodeUuid("FixedDatanodeUuid"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); DataStorage.createStorageID(sd, false); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); createBlockPoolVersionFile(bpDir, version, bpidToWrite); } }
public SimulatedFSDataset(DataNode datanode, DataStorage storage, Configuration conf) { this.datanode = datanode; if (storage != null) { for (int i = 0; i < storage.getNumStorageDirs(); ++i) { DataStorage.createStorageID(storage.getStorageDir(i), false); } this.datanodeUuid = storage.getDatanodeUuid(); } else { this.datanodeUuid = "SimulatedDatanode-" + DataNode.generateUuid(); } registerMBean(datanodeUuid); this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY), conf.getEnum(CONFIG_PROPERTY_STATE, DEFAULT_STATE)); this.volume = new SimulatedVolume(this.storage); }