try (AutoCloseableLock lock = datasetLock.acquire()) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final StorageLocation sdLocation = sd.getStorageLocation(); LOG.info("Checking removing StorageLocation " +
addVolume(storage.getStorageDir(idx));
synchronized (this) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final File absRoot = sd.getRoot().getAbsoluteFile(); if (volumesToRemove.contains(absRoot)) {
synchronized (this) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final File absRoot = sd.getRoot().getAbsoluteFile(); if (volumesToRemove.contains(absRoot)) {
private static void createStorageDirs(DataStorage storage, Configuration conf, int numDirs) throws IOException { List<Storage.StorageDirectory> dirs = new ArrayList<Storage.StorageDirectory>(); List<String> dirStrings = new ArrayList<String>(); for (int i = 0; i < numDirs; i++) { File loc = new File(BASE_DIR + "/data" + i); dirStrings.add(new Path(loc.toString()).toUri().toString()); loc.mkdirs(); dirs.add(createStorageDirectory(loc)); when(storage.getStorageDir(i)).thenReturn(dirs.get(i)); } String dataDir = StringUtils.join(",", dirStrings); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDir); when(storage.dirIterator()).thenReturn(dirs.iterator()); when(storage.getNumStorageDirs()).thenReturn(numDirs); }
/** * An FSDataset has a directory where it loads its data files. */ public FSDataset(DataStorage storage, Configuration conf) throws IOException { this.maxBlocksPerDir = conf.getInt("dfs.datanode.numblocks", 64); FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()]; for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { volArray[idx] = new FSVolume(storage.getStorageDir(idx).getCurrentDir(), conf); } volumes = new FSVolumeSet(volArray); volumeMap = new HashMap<Block, DatanodeBlockInfo>(); volumes.getVolumeMap(volumeMap); registerMBean(storage.getStorageID()); }
roots[idx] = storage.getStorageDir(idx).getCurrentDir(); volArray[idx] = new FSVolume(this, storage.getStorageDir(idx).getCurrentDir(), conf); DataNode.LOG.info("FSDataset added volume - " + storage.getStorageDir(idx).getCurrentDir());
doTransition(getStorageDir(idx), nsInfo, startOpt); assert this.getLayoutVersion() == nsInfo.getLayoutVersion() : "Data-node and name-node layout versions must be the same.";
private void doRollback(NamespaceInfo nsInfo) throws IOException { int numDirs = getNumStorageDirs(); RollbackThread[] rollbackThreads = new RollbackThread[numDirs]; // start to rollback for (int i=0; i<numDirs; i++) { final StorageDirectory sd = this.getStorageDir(i); RollbackThread thread = new RollbackThread(sd, nsInfo, new DataStorage( datanode)); thread.start(); rollbackThreads[i] = thread; } // wait for rollback to be done for (RollbackThread thread : rollbackThreads) { try { thread.join(); } catch (InterruptedException e) { return; } } // check for errors for (RollbackThread thread : rollbackThreads) { if (thread.error != null) throw new IOException(thread.error); } }
@Test(timeout = 5000) public void testRemoveNewlyAddedVolume() throws IOException { final int numExistingVolumes = dataset.getVolumes().size(); List<NamespaceInfo> nsInfos = new ArrayList<>(); for (String bpid : BLOCK_POOL_IDS) { nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1)); } String newVolumePath = BASE_DIR + "/newVolumeToRemoveLater"; StorageLocation loc = StorageLocation.parse(newVolumePath); Storage.StorageDirectory sd = createStorageDirectory(new File(newVolumePath)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), anyListOf(NamespaceInfo.class))) .thenReturn(builder); dataset.addVolume(loc, nsInfos); assertEquals(numExistingVolumes + 1, dataset.getVolumes().size()); when(storage.getNumStorageDirs()).thenReturn(numExistingVolumes + 1); when(storage.getStorageDir(numExistingVolumes)).thenReturn(sd); Set<File> volumesToRemove = new HashSet<>(); volumesToRemove.add(loc.getFile()); dataset.removeVolumes(volumesToRemove, true); assertEquals(numExistingVolumes, dataset.getVolumes().size()); }
Storage.StorageDirectory sd = storage.getStorageDir(i); assertThat(sd.getRoot().toString(), is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
addVolume(dataLocations, storage.getStorageDir(idx));
addVolume(dataLocations, storage.getStorageDir(idx));
public SimulatedFSDataset(DataNode datanode, DataStorage storage, Configuration conf) { this.datanode = datanode; if (storage != null) { for (int i = 0; i < storage.getNumStorageDirs(); ++i) { DataStorage.createStorageID(storage.getStorageDir(i), false); } this.datanodeUuid = storage.getDatanodeUuid(); } else { this.datanodeUuid = "SimulatedDatanode-" + DataNode.generateUuid(); } registerMBean(datanodeUuid); this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY), conf.getEnum(CONFIG_PROPERTY_STATE, DEFAULT_STATE)); this.volume = new SimulatedVolume(this.storage); }
assertEquals(1, storage.getNumStorageDirs()); for (int i = 0; i < storage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = storage.getStorageDir(i); assertFalse(sd.getRoot().getAbsolutePath().startsWith( dn0Vol1.getAbsolutePath()