public void submitSyncFileRangeRequest(FsVolumeImpl volume, final ReplicaOutputStreams streams, final long offset, final long nbytes, final int flags) { execute(volume, new Runnable() { @Override public void run() { try { streams.syncFileRangeIfPossible(offset, nbytes, flags); } catch (NativeIOException e) { LOG.warn("sync_file_range error", e); } } }); }
/** * Starts AsyncDiskService for a new volume * @param volume the root of the new data volume. */ synchronized void addVolume(FsVolumeImpl volume) { if (executors == null) { throw new RuntimeException("AsyncDiskService is already shutdown"); } if (volume == null) { throw new RuntimeException("Attempt to add a null volume"); } ThreadPoolExecutor executor = executors.get(volume.getStorageID()); if (executor != null) { throw new RuntimeException("Volume " + volume + " is already existed."); } addExecutorForVolume(volume); }
/** * Activate a volume to serve requests. * @throws IOException if the storage UUID already exists. */ private void activateVolume( ReplicaMap replicaMap, Storage.StorageDirectory sd, StorageType storageType, FsVolumeReference ref) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); if (dnStorage != null) { final String errorMsg = String.format( "Found duplicated storage UUID: %s in %s.", sd.getStorageUuid(), sd.getVersionFile()); LOG.error(errorMsg); throw new IOException(errorMsg); } volumeMap.addAll(replicaMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume()); volumes.addVolume(ref); } }
asyncDiskService.deleteAsync(v.obtainReference(), removing, new ExtendedBlock(bpid, invalidBlks[i]), dataStorage.getTrashDirectoryForReplica(bpid, removing)); } else { asyncDiskService.deleteSync(v.obtainReference(), removing, new ExtendedBlock(bpid, invalidBlks[i]), dataStorage.getTrashDirectoryForReplica(bpid, removing));
@Override public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block, ReplicaOutputStreams outs, long offset, long nbytes, int flags) { FsVolumeImpl fsVolumeImpl = this.getVolume(block); asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, outs, offset, nbytes, flags); }
@Override // FsDatasetSpi public void shutdown() { fsRunning = false; if (lazyWriter != null) { ((LazyWriter) lazyWriter.getRunnable()).stop(); lazyWriter.interrupt(); } if (mbeanName != null) { MBeans.unregister(mbeanName); } if (asyncDiskService != null) { asyncDiskService.shutdown(); } if (asyncLazyPersistService != null) { asyncLazyPersistService.shutdown(); } if(volumes != null) { volumes.shutdown(); } if (lazyWriter != null) { try { lazyWriter.join(); } catch (InterruptedException ie) { LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " + "from LazyWriter.join"); } } }
sd.getStorageUuid() + " from FsDataset."); asyncDiskService.removeVolume(sd.getStorageUuid()); volumes.removeVolume(sdLocation, clearFailure); volumes.waitVolumeRemoved(5000, datasetLockCondition);
volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(), blockChooserImpl); asyncDiskService = new FsDatasetAsyncDiskService(datanode, this); asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode, conf); deletingBlock = new HashMap<String, Set<Long>>();
asyncDiskService.deleteAsync(v.obtainReference(), f, FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()), new ExtendedBlock(bpid, invalidBlks[i]), dataStorage.getTrashDirectoryForBlockFile(bpid, f)); } else { asyncDiskService.deleteSync(v.obtainReference(), f, FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()), new ExtendedBlock(bpid, invalidBlks[i]),
@Override public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block, FileDescriptor fd, long offset, long nbytes, int flags) { FsVolumeImpl fsVolumeImpl = this.getVolume(block); asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, fd, offset, nbytes, flags); }
@Override // FsDatasetSpi public void shutdown() { fsRunning = false; ((LazyWriter) lazyWriter.getRunnable()).stop(); lazyWriter.interrupt(); if (mbeanName != null) { MBeans.unregister(mbeanName); } if (asyncDiskService != null) { asyncDiskService.shutdown(); } if (asyncLazyPersistService != null) { asyncLazyPersistService.shutdown(); } if(volumes != null) { volumes.shutdown(); } try { lazyWriter.join(); } catch (InterruptedException ie) { LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " + "from LazyWriter.join"); } }
asyncDiskService.removeVolume(sd.getCurrentDir()); volumes.removeVolume(absRoot, clearFailure);
volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(), blockChooserImpl); asyncDiskService = new FsDatasetAsyncDiskService(datanode, this); asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode); deletingBlock = new HashMap<String, Set<Long>>();
/** * Delete the block file and meta file from the disk asynchronously, adjust * dfsUsed statistics accordingly. */ void deleteAsync(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete, ExtendedBlock block, String trashDirectory) { LOG.info("Scheduling " + block.getLocalBlock() + " replica " + replicaToDelete + " for deletion"); ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask( volumeRef, replicaToDelete, block, trashDirectory); execute(((FsVolumeImpl) volumeRef.getVolume()), deletionTask); }
asyncDiskService.deleteAsync(v.obtainReference(), f, FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()), new ExtendedBlock(bpid, invalidBlks[i]), dataStorage.getTrashDirectoryForBlockFile(bpid, f)); } else { asyncDiskService.deleteSync(v.obtainReference(), f, FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()), new ExtendedBlock(bpid, invalidBlks[i]),
private void addVolume(Collection<StorageLocation> dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, sd.getRoot()); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImpl( this, sd.getStorageUuid(), dir, this.conf, storageType); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(this); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume(sd.getCurrentDir()); volumes.addVolume(ref); } LOG.info("Added volume - " + dir + ", StorageType: " + storageType); }
@Override public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block, FileDescriptor fd, long offset, long nbytes, int flags) { FsVolumeImpl fsVolumeImpl = this.getVolume(block); asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, fd, offset, nbytes, flags); }
/** * Starts AsyncDiskService for a new volume * @param volume the root of the new data volume. */ synchronized void addVolume(File volume) { if (executors == null) { throw new RuntimeException("AsyncDiskService is already shutdown"); } ThreadPoolExecutor executor = executors.get(volume); if (executor != null) { throw new RuntimeException("Volume " + volume + " is already existed."); } addExecutorForVolume(volume); }
@Override // FsDatasetSpi public void shutdown() { fsRunning = false; ((LazyWriter) lazyWriter.getRunnable()).stop(); lazyWriter.interrupt(); if (mbeanName != null) { MBeans.unregister(mbeanName); } if (asyncDiskService != null) { asyncDiskService.shutdown(); } if (asyncLazyPersistService != null) { asyncLazyPersistService.shutdown(); } if(volumes != null) { volumes.shutdown(); } try { lazyWriter.join(); } catch (InterruptedException ie) { LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " + "from LazyWriter.join"); } }
asyncDiskService.removeVolume(sd.getCurrentDir()); volumes.removeVolume(absRoot, clearFailure);