private StorageType getStorageTypeFromLocations( Collection<StorageLocation> dataLocations, File dir) { for (StorageLocation dataLocation : dataLocations) { if (dataLocation.getFile().equals(dir)) { return dataLocation.getStorageType(); } } return StorageType.DEFAULT; }
private StorageType getStorageTypeFromLocations( Collection<StorageLocation> dataLocations, File dir) { for (StorageLocation dataLocation : dataLocations) { if (dataLocation.getFile().equals(dir)) { return dataLocation.getStorageType(); } } return StorageType.DEFAULT; }
/** * Remove volumes from DataNode. * See {@link removeVolumes(final Set<File>, boolean)} for details. * * @param locations the StorageLocations of the volumes to be removed. * @throws IOException */ private void removeVolumes(final Collection<StorageLocation> locations) throws IOException { if (locations.isEmpty()) { return; } Set<File> volumesToRemove = new HashSet<>(); for (StorageLocation loc : locations) { volumesToRemove.add(loc.getFile().getAbsoluteFile()); } removeVolumes(volumesToRemove, true); }
/** * Remove volumes from DataNode. * See {@link removeVolumes(final Set<File>, boolean)} for details. * * @param locations the StorageLocations of the volumes to be removed. * @throws IOException */ private void removeVolumes(final Collection<StorageLocation> locations) throws IOException { if (locations.isEmpty()) { return; } Set<File> volumesToRemove = new HashSet<>(); for (StorageLocation loc : locations) { volumesToRemove.add(loc.getFile().getAbsoluteFile()); } removeVolumes(volumesToRemove, true); }
static List<StorageLocation> checkStorageLocations( Collection<StorageLocation> dataDirs, LocalFileSystem localFS, DataNodeDiskChecker dataNodeDiskChecker) throws IOException { ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>(); StringBuilder invalidDirs = new StringBuilder(); for (StorageLocation location : dataDirs) { final URI uri = location.getUri(); try { dataNodeDiskChecker.checkDir(localFS, new Path(uri)); locations.add(location); } catch (IOException ioe) { LOG.warn("Invalid " + DFS_DATANODE_DATA_DIR_KEY + " " + location.getFile() + " : ", ioe); invalidDirs.append("\"").append(uri.getPath()).append("\" "); } } if (locations.size() == 0) { throw new IOException("All directories in " + DFS_DATANODE_DATA_DIR_KEY + " are invalid: " + invalidDirs); } return locations; }
static List<StorageLocation> checkStorageLocations( Collection<StorageLocation> dataDirs, LocalFileSystem localFS, DataNodeDiskChecker dataNodeDiskChecker) throws IOException { ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>(); StringBuilder invalidDirs = new StringBuilder(); for (StorageLocation location : dataDirs) { final URI uri = location.getUri(); try { dataNodeDiskChecker.checkDir(localFS, new Path(uri)); locations.add(location); } catch (IOException ioe) { LOG.warn("Invalid " + DFS_DATANODE_DATA_DIR_KEY + " " + location.getFile() + " : ", ioe); invalidDirs.append("\"").append(uri.getPath()).append("\" "); } } if (locations.size() == 0) { throw new IOException("All directories in " + DFS_DATANODE_DATA_DIR_KEY + " are invalid: " + invalidDirs); } return locations; }
if (absoluteVolumePaths.contains(loc.getFile().getAbsoluteFile())) { it.remove();
/** * Gets initial volume failure information for all volumes that failed * immediately at startup. The method works by determining the set difference * between all configured storage locations and the actual storage locations in * use after attempting to put all of them into service. * * @return each storage location that has failed */ private static List<VolumeFailureInfo> getInitialVolumeFailureInfos( Collection<StorageLocation> dataLocations, DataStorage storage) { Set<String> failedLocationSet = Sets.newHashSetWithExpectedSize( dataLocations.size()); for (StorageLocation sl: dataLocations) { failedLocationSet.add(sl.getFile().getAbsolutePath()); } for (Iterator<Storage.StorageDirectory> it = storage.dirIterator(); it.hasNext(); ) { Storage.StorageDirectory sd = it.next(); failedLocationSet.remove(sd.getRoot().getAbsolutePath()); } List<VolumeFailureInfo> volumeFailureInfos = Lists.newArrayListWithCapacity( failedLocationSet.size()); long failureDate = Time.now(); for (String failedStorageLocation: failedLocationSet) { volumeFailureInfos.add(new VolumeFailureInfo(failedStorageLocation, failureDate)); } return volumeFailureInfos; }
sl.hasNext(); ) { StorageLocation location = sl.next(); if (location.getFile().getCanonicalPath().equals( dir.getRoot().getCanonicalPath())) { sl.remove();
sl.hasNext(); ) { StorageLocation location = sl.next(); if (location.getFile().getCanonicalPath().equals( dir.getRoot().getCanonicalPath())) { sl.remove();
/** * Gets initial volume failure information for all volumes that failed * immediately at startup. The method works by determining the set difference * between all configured storage locations and the actual storage locations in * use after attempting to put all of them into service. * * @return each storage location that has failed */ private static List<VolumeFailureInfo> getInitialVolumeFailureInfos( Collection<StorageLocation> dataLocations, DataStorage storage) { Set<String> failedLocationSet = Sets.newHashSetWithExpectedSize( dataLocations.size()); for (StorageLocation sl: dataLocations) { failedLocationSet.add(sl.getFile().getAbsolutePath()); } for (Iterator<Storage.StorageDirectory> it = storage.dirIterator(); it.hasNext(); ) { Storage.StorageDirectory sd = it.next(); failedLocationSet.remove(sd.getRoot().getAbsolutePath()); } List<VolumeFailureInfo> volumeFailureInfos = Lists.newArrayListWithCapacity( failedLocationSet.size()); long failureDate = Time.now(); for (String failedStorageLocation: failedLocationSet) { volumeFailureInfos.add(new VolumeFailureInfo(failedStorageLocation, failureDate)); } return volumeFailureInfos; }
File lockFile = new File(sl.getFile(), Storage.STORAGE_FILE_LOCK); try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rws"); FileChannel channel = raf.getChannel()) {
@Test public void testParseChangedVolumes() throws IOException { startDFSCluster(1, 1); DataNode dn = cluster.getDataNodes().get(0); Configuration conf = dn.getConf(); String oldPaths = conf.get(DFS_DATANODE_DATA_DIR_KEY); List<StorageLocation> oldLocations = new ArrayList<StorageLocation>(); for (String path : oldPaths.split(",")) { oldLocations.add(StorageLocation.parse(path)); } assertFalse(oldLocations.isEmpty()); String newPaths = oldLocations.get(0).getFile().getAbsolutePath() + ",/foo/path1,/foo/path2"; DataNode.ChangedVolumes changedVolumes = dn.parseChangedVolumes(newPaths); List<StorageLocation> newVolumes = changedVolumes.newLocations; assertEquals(2, newVolumes.size()); assertEquals(new File("/foo/path1").getAbsolutePath(), newVolumes.get(0).getFile().getAbsolutePath()); assertEquals(new File("/foo/path2").getAbsolutePath(), newVolumes.get(1).getFile().getAbsolutePath()); List<StorageLocation> removedVolumes = changedVolumes.deactivateLocations; assertEquals(1, removedVolumes.size()); assertEquals(oldLocations.get(1).getFile(), removedVolumes.get(0).getFile()); assertEquals(1, changedVolumes.unchangedLocations.size()); assertEquals(oldLocations.get(0).getFile(), changedVolumes.unchangedLocations.get(0).getFile()); }
@Test (timeout = 30000) public void testDataDirValidation() throws Throwable { DataNodeDiskChecker diskChecker = mock(DataNodeDiskChecker.class); doThrow(new IOException()).doThrow(new IOException()).doNothing() .when(diskChecker).checkDir(any(LocalFileSystem.class), any(Path.class)); LocalFileSystem fs = mock(LocalFileSystem.class); AbstractList<StorageLocation> locations = new ArrayList<StorageLocation>(); locations.add(StorageLocation.parse("file:/p1/")); locations.add(StorageLocation.parse("file:/p2/")); locations.add(StorageLocation.parse("file:/p3/")); List<StorageLocation> checkedLocations = DataNode.checkStorageLocations(locations, fs, diskChecker); assertEquals("number of valid data dirs", 1, checkedLocations.size()); String validDir = checkedLocations.iterator().next().getFile().getPath(); assertThat("p3 should be valid", new File("/p3/").getPath(), is(validDir)); } }
final List<NamespaceInfo> nsInfos) throws IOException { final File dir = location.getFile(); builder = dataStorage.prepareVolume(datanode, location.getFile(), nsInfos); } catch (IOException e) { volumes.addVolumeFailureInfo(new VolumeFailureInfo( location.getFile().getAbsolutePath(), Time.now())); throw e;
@Test(timeout = 5000) public void testRemoveNewlyAddedVolume() throws IOException { final int numExistingVolumes = dataset.getVolumes().size(); List<NamespaceInfo> nsInfos = new ArrayList<>(); for (String bpid : BLOCK_POOL_IDS) { nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1)); } String newVolumePath = BASE_DIR + "/newVolumeToRemoveLater"; StorageLocation loc = StorageLocation.parse(newVolumePath); Storage.StorageDirectory sd = createStorageDirectory(new File(newVolumePath)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), anyListOf(NamespaceInfo.class))) .thenReturn(builder); dataset.addVolume(loc, nsInfos); assertEquals(numExistingVolumes + 1, dataset.getVolumes().size()); when(storage.getNumStorageDirs()).thenReturn(numExistingVolumes + 1); when(storage.getStorageDir(numExistingVolumes)).thenReturn(sd); Set<File> volumesToRemove = new HashSet<>(); volumesToRemove.add(loc.getFile()); dataset.removeVolumes(volumesToRemove, true); assertEquals(numExistingVolumes, dataset.getVolumes().size()); }
DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), anyListOf(NamespaceInfo.class))) .thenReturn(builder);
final List<NamespaceInfo> nsInfos) throws IOException { final File dir = location.getFile(); builder = dataStorage.prepareVolume(datanode, location.getFile(), nsInfos); } catch (IOException e) { volumes.addVolumeFailureInfo(new VolumeFailureInfo( location.getFile().getAbsolutePath(), Time.now())); throw e;
final String volumePathToRemove = dataDirs[0]; Set<File> volumesToRemove = new HashSet<>(); volumesToRemove.add(StorageLocation.parse(volumePathToRemove).getFile());
@Test public void testAddStorageDirectories() throws IOException, URISyntaxException { final int numLocations = 3; final int numNamespace = 3; List<StorageLocation> locations = createStorageLocations(numLocations); // Add volumes for multiple namespaces. List<NamespaceInfo> namespaceInfos = createNamespaceInfos(numNamespace); for (NamespaceInfo ni : namespaceInfos) { storage.addStorageLocations(mockDN, ni, locations, START_OPT); for (StorageLocation sl : locations) { checkDir(sl.getFile()); checkDir(sl.getFile(), ni.getBlockPoolID()); } } assertEquals(numLocations, storage.getNumStorageDirs()); locations = createStorageLocations(numLocations); List<StorageDirectory> addedLocation = storage.addStorageLocations(mockDN, namespaceInfos.get(0), locations, START_OPT); assertTrue(addedLocation.isEmpty()); // The number of active storage dirs has not changed, since it tries to // add the storage dirs that are under service. assertEquals(numLocations, storage.getNumStorageDirs()); // Add more directories. locations = createStorageLocations(6); storage.addStorageLocations(mockDN, nsInfo, locations, START_OPT); assertEquals(6, storage.getNumStorageDirs()); }