ReplicaFileDeleteTask(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete, ExtendedBlock block, String trashDirectory) { this.volumeRef = volumeRef; this.volume = (FsVolumeImpl) volumeRef.getVolume(); this.replicaToDelete = replicaToDelete; this.block = block; this.trashDirectory = trashDirectory; }
private void markHealthy() { synchronized (DatasetVolumeChecker.this) { healthyVolumes.add(reference.getVolume()); } }
/** * Get the volume for a given index. */ public FsVolumeSpi get(int index) { return references.get(index).getVolume(); }
private void markFailed() { synchronized (DatasetVolumeChecker.this) { failedVolumes.add(reference.getVolume()); } }
@Override public FsVolumeSpi next() { int refIdx = idx++; return references.get(refIdx).getVolume(); }
@Override public void onFailure(@Nonnull Throwable t) { Throwable exception = (t instanceof ExecutionException) ? t.getCause() : t; LOG.warn("Exception running disk checks against volume " + reference.getVolume(), exception); markFailed(); cleanup(); }
/** * Create a new temporary replica of replicaInfo object in specified volume. * * @param block - Extended Block * @param replicaInfo - ReplicaInfo * @param volumeRef - Volume Ref - Closed by caller. * @return newReplicaInfo new replica object created in specified volume. * @throws IOException */ @VisibleForTesting ReplicaInfo copyReplicaToVolume(ExtendedBlock block, ReplicaInfo replicaInfo, FsVolumeReference volumeRef) throws IOException { FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume(); // Copy files to temp dir first ReplicaInfo newReplicaInfo = targetVolume.moveBlockToTmpLocation(block, replicaInfo, smallBufferSize, conf); return newReplicaInfo; }
@Override public void onSuccess(@Nonnull VolumeCheckResult result) { switch(result) { case HEALTHY: case DEGRADED: LOG.debug("Volume {} is {}.", reference.getVolume(), result); markHealthy(); break; case FAILED: LOG.warn("Volume {} detected as being unhealthy", reference.getVolume()); markFailed(); break; default: LOG.error("Unexpected health check result {} for volume {}", result, reference.getVolume()); markHealthy(); break; } cleanup(); }
public void dropCacheBehindReads(String identifier, long offset, long len, int flags) throws NativeIOException { assert this.dataInFd != null : "null dataInFd!"; fileIoProvider.posixFadvise(getVolumeRef().getVolume(), identifier, dataInFd, offset, len, flags); }
/** * Delete the block file and meta file from the disk asynchronously, adjust * dfsUsed statistics accordingly. */ void deleteAsync(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete, ExtendedBlock block, String trashDirectory) { LOG.info("Scheduling " + block.getLocalBlock() + " replica " + replicaToDelete + " for deletion"); ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask( volumeRef, replicaToDelete, block, trashDirectory); execute(((FsVolumeImpl) volumeRef.getVolume()), deletionTask); }
/** * Asynchronously lazy persist the block from the RamDisk to Disk. */ void submitLazyPersistTask(String bpId, long blockId, long genStamp, long creationTime, ReplicaInfo replica, FsVolumeReference target) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("LazyWriter schedule async task to persist RamDisk block pool id: " + bpId + " block id: " + blockId); } ReplicaLazyPersistTask lazyPersistTask = new ReplicaLazyPersistTask( bpId, blockId, genStamp, creationTime, replica, target); FsVolumeImpl volume = (FsVolumeImpl)target.getVolume(); execute(volume.getStorageID(), lazyPersistTask); }
VolumeScanner(Conf conf, DataNode datanode, FsVolumeReference ref) { this.conf = conf; this.datanode = datanode; this.metrics = datanode.getMetrics(); this.ref = ref; this.volume = ref.getVolume(); ScanResultHandler handler; try { handler = conf.resultHandler.newInstance(); } catch (Throwable e) { LOG.error("unable to instantiate {}", conf.resultHandler, e); handler = new ScanResultHandler(); } this.resultHandler = handler; setName("VolumeScannerThread(" + volume + ")"); setDaemon(true); }
final FsVolumeReference reference = references.getReference(i); Optional<ListenableFuture<VolumeCheckResult>> olf = delegateChecker.schedule(reference.getVolume(), IGNORED_CONTEXT); LOG.info("Scheduled health check for volume {}", reference.getVolume()); if (olf.isPresent()) { allVolumes.add(reference.getVolume()); Futures.addCallback(olf.get(), new ResultHandler(reference, healthyVolumes, failedVolumes,
/** * Dynamically add new volumes to the existing volumes that this DN manages. * * @param ref a reference to the new FsVolumeImpl instance. */ void addVolume(FsVolumeReference ref) { FsVolumeImpl volume = (FsVolumeImpl) ref.getVolume(); volumes.add(volume); if (blockScanner != null) { blockScanner.addVolumeScanner(ref); } else { // If the volume is not put into a volume scanner, it does not need to // hold the reference. IOUtils.cleanup(null, ref); } // If the volume is used to replace a failed volume, it needs to reset the // volume failure info for this volume. removeVolumeFailureInfo(volume.getStorageLocation()); FsDatasetImpl.LOG.info("Added new volume: " + volume.getStorageID()); }
boolean success = false; try { FsVolumeSpi volume = ref.getVolume(); if (!isEnabled()) { LOG.debug("Not adding volume scanner for {}, because the block " +
/** * Activate a volume to serve requests. * @throws IOException if the storage UUID already exists. */ private void activateVolume( ReplicaMap replicaMap, Storage.StorageDirectory sd, StorageType storageType, FsVolumeReference ref) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); if (dnStorage != null) { final String errorMsg = String.format( "Found duplicated storage UUID: %s in %s.", sd.getStorageUuid(), sd.getVersionFile()); LOG.error(errorMsg); throw new IOException(errorMsg); } volumeMap.addAll(replicaMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume()); volumes.addVolume(ref); } }
@Override public void run() { boolean succeeded = false; final FsDatasetImpl dataset = (FsDatasetImpl)datanode.getFSDataset(); try (FsVolumeReference ref = this.targetVolume) { int smallBufferSize = DFSUtilClient.getSmallBufferSize(EMPTY_HDFS_CONF); FsVolumeImpl volume = (FsVolumeImpl)ref.getVolume(); File[] targetFiles = volume.copyBlockToLazyPersistLocation(bpId, blockId, genStamp, replicaInfo, smallBufferSize, conf); // Lock FsDataSetImpl during onCompleteLazyPersist callback dataset.onCompleteLazyPersist(bpId, blockId, creationTime, targetFiles, volume); succeeded = true; } catch (Exception e){ FsDatasetImpl.LOG.warn( "LazyWriter failed to async persist RamDisk block pool id: " + bpId + "block Id: " + blockId, e); } finally { if (!succeeded) { dataset.onFailLazyPersist(bpId, blockId); } } } }
targetVolume = (FsVolumeImpl) targetReference.getVolume();
FsVolumeReference ref = volumes.getNextVolume(storageType, storageId, b .getNumBytes()); FsVolumeImpl v = (FsVolumeImpl) ref.getVolume(); ReplicaInPipeline newReplicaInfo; try {
FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();