long getCapacity() { long capacity = 0L; for (FsVolumeImpl v : volumes) { try (FsVolumeReference ref = v.obtainReference()) { capacity += v.getCapacity(); } catch (IOException e) { // ignore. } } return capacity; }
long getDfsUsed() throws IOException { long dfsUsed = 0L; for (FsVolumeImpl v : volumes) { try(FsVolumeReference ref = v.obtainReference()) { dfsUsed += v.getDfsUsed(); } catch (ClosedChannelException e) { // ignore. } } return dfsUsed; }
long getBlockPoolUsed(String bpid) throws IOException { long dfsUsed = 0L; for (FsVolumeImpl v : volumes) { try (FsVolumeReference ref = v.obtainReference()) { dfsUsed += v.getBlockPoolUsed(bpid); } catch (ClosedChannelException e) { // ignore. } } return dfsUsed; }
private FsVolumeReference chooseVolume(List<FsVolumeImpl> list, long blockSize, String storageId) throws IOException { while (true) { FsVolumeImpl volume = blockChooser.chooseVolume(list, blockSize, storageId); try { return volume.obtainReference(); } catch (ClosedChannelException e) { FsDatasetImpl.LOG.warn("Chosen a closed volume: " + volume); // blockChooser.chooseVolume returns DiskOutOfSpaceException when the list // is empty, indicating that all volumes are closed. list.remove(volume); } } }
public void run() { try (FsVolumeReference ref = v.obtainReference()) { FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid + " on volume " + v + "..."); long startTime = Time.monotonicNow(); v.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap); long timeTaken = Time.monotonicNow() - startTime; FsDatasetImpl.LOG.info("Time to add replicas to map for block pool" + " " + bpid + " on volume " + v + ": " + timeTaken + "ms"); } catch (ClosedChannelException e) { FsDatasetImpl.LOG.info("The volume " + v + " is closed while " + "adding replicas, ignored."); } catch (IOException ioe) { FsDatasetImpl.LOG.info("Caught exception while adding replicas " + "from " + v + ". Will throw later.", ioe); exceptions.add(ioe); } } };
public void run() { try (FsVolumeReference ref = v.obtainReference()) { FsDatasetImpl.LOG.info("Scanning block pool " + bpid + " on volume " + v + "..."); long startTime = Time.monotonicNow(); v.addBlockPool(bpid, conf); long timeTaken = Time.monotonicNow() - startTime; FsDatasetImpl.LOG.info("Time taken to scan block pool " + bpid + " on " + v + ": " + timeTaken + "ms"); } catch (ClosedChannelException e) { // ignore. } catch (IOException ioe) { FsDatasetImpl.LOG.info("Caught exception while scanning " + v + ". Will throw later.", ioe); exceptions.add(ioe); } } };
private Collection<VolumeInfo> getVolumeInfo() { Collection<VolumeInfo> info = new ArrayList<VolumeInfo>(); for (FsVolumeImpl volume : volumes.getVolumes()) { long used = 0; long free = 0; try (FsVolumeReference ref = volume.obtainReference()) { used = volume.getDfsUsed(); free = volume.getAvailable(); } catch (ClosedChannelException e) { continue; } catch (IOException e) { LOG.warn(e.getMessage()); used = 0; free = 0; } info.add(new VolumeInfo(volume, used, free)); } return info; }
@Override //FsDatasetSpi public void deleteBlockPool(String bpid, boolean force) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { List<FsVolumeImpl> curVolumes = volumes.getVolumes(); if (!force) { for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { if (!volume.isBPDirEmpty(bpid)) { LOG.warn(bpid + " has some block files, cannot delete unless forced"); throw new IOException("Cannot delete block pool, " + "it contains some block files"); } } catch (ClosedChannelException e) { // ignore. } } } for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { volume.deleteBPDirectories(bpid, force); } catch (ClosedChannelException e) { // ignore. } } } }
/** * Updates the failed volume info in the volumeFailureInfos Map * and calls {@link #removeVolume(FsVolumeImpl)} to remove the volume * from the volume list for each of the failed volumes. * * @param failedVolumes set of volumes marked failed. */ void handleVolumeFailures(Set<FsVolumeSpi> failedVolumes) { try (AutoCloseableLock lock = checkDirsLock.acquire()) { for(FsVolumeSpi vol : failedVolumes) { FsVolumeImpl fsv = (FsVolumeImpl) vol; try (FsVolumeReference ref = fsv.obtainReference()) { addVolumeFailureInfo(fsv); removeVolume(fsv); } catch (ClosedChannelException e) { FsDatasetImpl.LOG.debug("Caught exception when obtaining " + "reference count on closed volume", e); } catch (IOException e) { FsDatasetImpl.LOG.error("Unexpected IOException", e); } } waitVolumeRemoved(5000, checkDirsLockCondition); } }
@Override // FsDatasetSpi public StorageReport[] getStorageReports(String bpid) throws IOException { List<StorageReport> reports; // Volumes are the references from a copy-on-write snapshot, so the // access on the volume metrics doesn't require an additional lock. List<FsVolumeImpl> curVolumes = volumes.getVolumes(); reports = new ArrayList<>(curVolumes.size()); for (FsVolumeImpl volume : curVolumes) { try (FsVolumeReference ref = volume.obtainReference()) { StorageReport sr = new StorageReport(volume.toDatanodeStorage(), false, volume.getCapacity(), volume.getDfsUsed(), volume.getAvailable(), volume.getBlockPoolUsed(bpid), volume.getNonDfsUsed()); reports.add(sr); } catch (ClosedChannelException e) { continue; } } return reports.toArray(new StorageReport[reports.size()]); }
volume, blockFile); ReplicaInputStreams ris = new ReplicaInputStreams(blockIn, checksumIn, volume.obtainReference(), fileIoProvider)) { ris.skipChecksumFully((numChunks - 1) * checksumSize); long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum;
asyncDiskService.deleteAsync(v.obtainReference(), removing, new ExtendedBlock(bpid, invalidBlks[i]), dataStorage.getTrashDirectoryForReplica(bpid, removing)); } else { asyncDiskService.deleteSync(v.obtainReference(), removing, new ExtendedBlock(bpid, invalidBlks[i]), dataStorage.getTrashDirectoryForReplica(bpid, removing));
final FsVolumeReference ref = fsVolume.obtainReference(); setupAsyncLazyPersistThread(fsVolume);
private void addVolume(Storage.StorageDirectory sd) throws IOException { final StorageLocation storageLocation = sd.getStorageLocation(); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImplBuilder() .setDataset(this) .setStorageID(sd.getStorageUuid()) .setStorageDirectory(sd) .setFileIoProvider(datanode.getFileIoProvider()) .setConf(this.conf) .build(); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(datasetLock); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); activateVolume(tempVolumeMap, sd, storageLocation.getStorageType(), ref); LOG.info("Added volume - " + storageLocation + ", StorageType: " + storageLocation.getStorageType()); }
long getBlockPoolUsed(String bpid) throws IOException { long dfsUsed = 0L; for (FsVolumeImpl v : volumes.get()) { try (FsVolumeReference ref = v.obtainReference()) { dfsUsed += v.getBlockPoolUsed(bpid); } catch (ClosedChannelException e) { // ignore. } } return dfsUsed; }
long getDfsUsed() throws IOException { long dfsUsed = 0L; for (FsVolumeImpl v : volumes.get()) { try(FsVolumeReference ref = v.obtainReference()) { dfsUsed += v.getDfsUsed(); } catch (ClosedChannelException e) { // ignore. } } return dfsUsed; }
long getCapacity() { long capacity = 0L; for (FsVolumeImpl v : volumes.get()) { try (FsVolumeReference ref = v.obtainReference()) { capacity += v.getCapacity(); } catch (IOException e) { // ignore. } } return capacity; }
long getBlockPoolUsed(String bpid) throws IOException { long dfsUsed = 0L; for (FsVolumeImpl v : volumes.get()) { try (FsVolumeReference ref = v.obtainReference()) { dfsUsed += v.getBlockPoolUsed(bpid); } catch (ClosedChannelException e) { // ignore. } } return dfsUsed; }
long getDfsUsed() throws IOException { long dfsUsed = 0L; for (FsVolumeImpl v : volumes.get()) { try(FsVolumeReference ref = v.obtainReference()) { dfsUsed += v.getDfsUsed(); } catch (ClosedChannelException e) { // ignore. } } return dfsUsed; }
long getCapacity() { long capacity = 0L; for (FsVolumeImpl v : volumes.get()) { try (FsVolumeReference ref = v.obtainReference()) { capacity += v.getCapacity(); } catch (IOException e) { // ignore. } } return capacity; }