List<String> storageToRemove = new ArrayList<>(); try (AutoCloseableLock lock = datasetLock.acquire()) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final StorageLocation sdLocation = sd.getStorageLocation();
throw new DiskErrorException( "Too many failed volumes - " + "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " + volsConfigured + ", volumes failed: " + volsFailed + ", volume failures tolerated: " + volFailuresTolerated); throw new DiskErrorException( "Too many failed volumes - " + "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " + volsConfigured + ", volumes failed: " + volsFailed + ", volume failures tolerated: " + volFailuresTolerated); deletingBlock = new HashMap<String, Set<Long>>(); for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { addVolume(storage.getStorageDir(idx));
/** * An FSDataset has a directory where it loads its data files. */ public FSDataset(DataStorage storage, Configuration conf) throws IOException { this.maxBlocksPerDir = conf.getInt("dfs.datanode.numblocks", 64); FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()]; for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { volArray[idx] = new FSVolume(storage.getStorageDir(idx).getCurrentDir(), conf); } volumes = new FSVolumeSet(volArray); volumeMap = new HashMap<Block, DatanodeBlockInfo>(); volumes.getVolumeMap(volumeMap); registerMBean(storage.getStorageID()); }
int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length; this.validVolsRequired = volsConfigured - volFailuresTolerated; if (validVolsRequired < 1 || validVolsRequired > storage.getNumStorageDirs()) { throw new DiskErrorException("Too many failed volumes - " + "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " + volsConfigured + ", volume failures tolerated: " + volFailuresTolerated ); File[] roots = new File[storage.getNumStorageDirs()]; for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { roots[idx] = storage.getStorageDir(idx).getCurrentDir(); FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()]; for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { volArray[idx] = new FSVolume(this, storage.getStorageDir(idx).getCurrentDir(), conf);
List<String> storageToRemove = new ArrayList<>(); synchronized (this) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final File absRoot = sd.getRoot().getAbsoluteFile();
List<String> storageToRemove = new ArrayList<>(); synchronized (this) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final File absRoot = sd.getRoot().getAbsoluteFile();
private static void createStorageDirs(DataStorage storage, Configuration conf, int numDirs) throws IOException { List<Storage.StorageDirectory> dirs = new ArrayList<Storage.StorageDirectory>(); List<String> dirStrings = new ArrayList<String>(); for (int i = 0; i < numDirs; i++) { File loc = new File(BASE_DIR + "/data" + i); dirStrings.add(new Path(loc.toString()).toUri().toString()); loc.mkdirs(); dirs.add(createStorageDirectory(loc)); when(storage.getStorageDir(i)).thenReturn(dirs.get(i)); } String dataDir = StringUtils.join(",", dirStrings); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDir); when(storage.dirIterator()).thenReturn(dirs.iterator()); when(storage.getNumStorageDirs()).thenReturn(numDirs); }
for(int idx = 0; idx < getNumStorageDirs(); idx++) { doTransition(getStorageDir(idx), nsInfo, startOpt); assert this.getLayoutVersion() == nsInfo.getLayoutVersion() :
private void doRollback(NamespaceInfo nsInfo) throws IOException { int numDirs = getNumStorageDirs(); RollbackThread[] rollbackThreads = new RollbackThread[numDirs]; // start to rollback for (int i=0; i<numDirs; i++) { final StorageDirectory sd = this.getStorageDir(i); RollbackThread thread = new RollbackThread(sd, nsInfo, new DataStorage( datanode)); thread.start(); rollbackThreads[i] = thread; } // wait for rollback to be done for (RollbackThread thread : rollbackThreads) { try { thread.join(); } catch (InterruptedException e) { return; } } // check for errors for (RollbackThread thread : rollbackThreads) { if (thread.error != null) throw new IOException(thread.error); } }
+ "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " + volsConfigured + ", volumes failed: " + volsFailed deletingBlock = new HashMap<String, Set<Long>>(); for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { addVolume(dataLocations, storage.getStorageDir(idx));
+ "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " + volsConfigured + ", volumes failed: " + volsFailed deletingBlock = new HashMap<String, Set<Long>>(); for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { addVolume(dataLocations, storage.getStorageDir(idx));
@Test(timeout = 5000) public void testRemoveNewlyAddedVolume() throws IOException { final int numExistingVolumes = dataset.getVolumes().size(); List<NamespaceInfo> nsInfos = new ArrayList<>(); for (String bpid : BLOCK_POOL_IDS) { nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1)); } String newVolumePath = BASE_DIR + "/newVolumeToRemoveLater"; StorageLocation loc = StorageLocation.parse(newVolumePath); Storage.StorageDirectory sd = createStorageDirectory(new File(newVolumePath)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), anyListOf(NamespaceInfo.class))) .thenReturn(builder); dataset.addVolume(loc, nsInfos); assertEquals(numExistingVolumes + 1, dataset.getVolumes().size()); when(storage.getNumStorageDirs()).thenReturn(numExistingVolumes + 1); when(storage.getStorageDir(numExistingVolumes)).thenReturn(sd); Set<File> volumesToRemove = new HashSet<>(); volumesToRemove.add(loc.getFile()); dataset.removeVolumes(volumesToRemove, true); assertEquals(numExistingVolumes, dataset.getVolumes().size()); }
for (int i = 0; i < storage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = storage.getStorageDir(i); assertThat(sd.getRoot().toString(),
@Test public void testRecoverTransitionReadFailure() throws IOException { final int numLocations = 3; List<StorageLocation> locations = createStorageLocations(numLocations, true); try { storage.recoverTransitionRead(mockDN, nsInfo, locations, START_OPT); fail("An IOException should throw: all StorageLocations are NON_EXISTENT"); } catch (IOException e) { GenericTestUtils.assertExceptionContains( "All specified directories are failed to load.", e); } assertEquals(0, storage.getNumStorageDirs()); }
@Test public void testAddStorageDirectories() throws IOException, URISyntaxException { final int numLocations = 3; final int numNamespace = 3; List<StorageLocation> locations = createStorageLocations(numLocations); // Add volumes for multiple namespaces. List<NamespaceInfo> namespaceInfos = createNamespaceInfos(numNamespace); for (NamespaceInfo ni : namespaceInfos) { storage.addStorageLocations(mockDN, ni, locations, START_OPT); for (StorageLocation sl : locations) { checkDir(sl.getFile()); checkDir(sl.getFile(), ni.getBlockPoolID()); } } assertEquals(numLocations, storage.getNumStorageDirs()); locations = createStorageLocations(numLocations); List<StorageDirectory> addedLocation = storage.addStorageLocations(mockDN, namespaceInfos.get(0), locations, START_OPT); assertTrue(addedLocation.isEmpty()); // The number of active storage dirs has not changed, since it tries to // add the storage dirs that are under service. assertEquals(numLocations, storage.getNumStorageDirs()); // Add more directories. locations = createStorageLocations(6); storage.addStorageLocations(mockDN, nsInfo, locations, START_OPT); assertEquals(6, storage.getNumStorageDirs()); }
public SimulatedFSDataset(DataNode datanode, DataStorage storage, Configuration conf) { this.datanode = datanode; if (storage != null) { for (int i = 0; i < storage.getNumStorageDirs(); ++i) { DataStorage.createStorageID(storage.getStorageDir(i), false); } this.datanodeUuid = storage.getDatanodeUuid(); } else { this.datanodeUuid = "SimulatedDatanode-" + DataNode.generateUuid(); } registerMBean(datanodeUuid); this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY), conf.getEnum(CONFIG_PROPERTY_STATE, DEFAULT_STATE)); this.volume = new SimulatedVolume(this.storage); }
/** * This test enforces the behavior that if there is an exception from * doTransition() during DN starts up, the storage directories that have * already been processed are still visible, i.e., in * DataStorage.storageDirs(). */ @Test public void testRecoverTransitionReadDoTransitionFailure() throws IOException { final int numLocations = 3; List<StorageLocation> locations = createStorageLocations(numLocations); // Prepare volumes storage.recoverTransitionRead(mockDN, nsInfo, locations, START_OPT); assertEquals(numLocations, storage.getNumStorageDirs()); // Reset DataStorage storage.unlockAll(); storage = new DataStorage(); // Trigger an exception from doTransition(). nsInfo.clusterID = "cluster1"; try { storage.recoverTransitionRead(mockDN, nsInfo, locations, START_OPT); fail("Expect to throw an exception from doTransition()"); } catch (IOException e) { GenericTestUtils.assertExceptionContains("All specified directories", e); } assertEquals(0, storage.getNumStorageDirs()); } }
assertEquals(1, storage.getNumStorageDirs()); for (int i = 0; i < storage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = storage.getStorageDir(i); assertFalse(sd.getRoot().getAbsolutePath().startsWith(