new TracerConfigurationManager(DATANODE_HTRACE_PREFIX, conf); this.fileIoProvider = new FileIoProvider(conf, this); this.blockScanner = new BlockScanner(this); this.lastDiskErrorCheck = 0; this.maxNumberOfBlocksToLog = conf.getLong(DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
public BlockScanner(DataNode datanode, Configuration conf) { this.datanode = datanode; this.conf = new Conf(conf); if (isEnabled()) { LOG.info("Initialized block scanner with targetBytesPerSec {}", this.conf.targetBytesPerSec); } else { LOG.info("Disabled block scanner."); } }
if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) { LOG.error("BlockSender.sendChunks() exception: ", e); datanode.getBlockScanner().markSuspectBlock( ris.getVolumeRef().getVolume().getStorageID(), block);
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { response.setContentType("text/plain"); DataNode datanode = (DataNode) getServletContext().getAttribute("datanode"); BlockScanner blockScanner = datanode.getBlockScanner(); StringBuilder buffer = new StringBuilder(8 * 1024); if (!blockScanner.isEnabled()) { LOG.warn("Periodic block scanner is not running"); buffer.append("Periodic block scanner is not running. " + "Please check the datanode log if this is unexpected."); } else { buffer.append("Block Scanner Statistics\n\n"); blockScanner.printStats(buffer); } String resp = buffer.toString(); LOG.trace("Returned Servlet info {}", resp); response.getWriter().write(resp); } }
Mockito.when(mockScanner.isEnabled()).thenReturn(false); dn0.setBlockScanner(mockScanner); Path filePath = new Path("test.dat"); Mockito.verify(mockScanner).markSuspectBlock(Mockito.eq(storageId), Mockito.eq(block));
private void shutdownPeriodicScanners() { shutdownDirectoryScanner(); blockScanner.removeAllVolumeScanners(); }
initialFileLength+32*1024); BlockScanner.Conf newConf = new BlockScanner.Conf(conf); ctx.datanode.getBlockScanner().setConf(newConf); ctx.datanode.getBlockScanner().markSuspectBlock(storageID, first);
/** * Remove the given block pool from the block scanner, dataset, and storage. */ void shutdownBlockPool(BPOfferService bpos) { blockPoolManager.remove(bpos); if (bpos.hasBlockPoolId()) { // Possible that this is shutting down before successfully // registering anywhere. If that's the case, we wouldn't have // a block pool id String bpId = bpos.getBlockPoolId(); blockScanner.disableBlockPoolId(bpId); if (data != null) { data.shutdownBlockPool(bpId); } if (storage != null) { storage.removeBlockPoolStorage(bpId); } } }
/** * Dynamically add new volumes to the existing volumes that this DN manages. * * @param ref a reference to the new FsVolumeImpl instance. */ void addVolume(FsVolumeReference ref) { FsVolumeImpl volume = (FsVolumeImpl) ref.getVolume(); volumes.add(volume); if (blockScanner != null) { blockScanner.addVolumeScanner(ref); } else { // If the volume is not put into a volume scanner, it does not need to // hold the reference. IOUtils.cleanup(null, ref); } // If the volume is used to replace a failed volume, it needs to reset the // volume failure info for this volume. removeVolumeFailureInfo(volume.getStorageLocation()); FsDatasetImpl.LOG.info("Added new volume: " + volume.getStorageID()); }
/** * Dynamically remove a volume in the list. * @param target the volume instance to be removed. */ private void removeVolume(FsVolumeImpl target) { if (volumes.remove(target)) { if (blockScanner != null) { blockScanner.removeVolumeScanner(target); } try { target.setClosed(); } catch (IOException e) { FsDatasetImpl.LOG.warn( "Error occurs when waiting volume to close: " + target, e); } target.shutdown(); volumesBeingRemoved.add(target); FsDatasetImpl.LOG.info("Removed volume: " + target); } else { if (FsDatasetImpl.LOG.isDebugEnabled()) { FsDatasetImpl.LOG.debug("Volume " + target + " does not exist or is removed by others."); } } }
blockScanner.enableBlockPoolId(bpos.getBlockPoolId()); initDirectoryScanner(getConf()); initDiskBalancer(data, getConf());
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { response.setContentType("text/plain"); DataNode datanode = (DataNode) getServletContext().getAttribute("datanode"); BlockScanner blockScanner = datanode.getBlockScanner(); StringBuilder buffer = new StringBuilder(8 * 1024); if (!blockScanner.isEnabled()) { LOG.warn("Periodic block scanner is not running"); buffer.append("Periodic block scanner is not running. " + "Please check the datanode log if this is unexpected."); } else { buffer.append("Block Scanner Statistics\n\n"); blockScanner.printStats(buffer); } String resp = buffer.toString(); LOG.trace("Returned Servlet info {}", resp); response.getWriter().write(resp); } }
private void shutdownPeriodicScanners() { shutdownDirectoryScanner(); blockScanner.removeAllVolumeScanners(); }
/** * Remove the given block pool from the block scanner, dataset, and storage. */ void shutdownBlockPool(BPOfferService bpos) { blockPoolManager.remove(bpos); if (bpos.hasBlockPoolId()) { // Possible that this is shutting down before successfully // registering anywhere. If that's the case, we wouldn't have // a block pool id String bpId = bpos.getBlockPoolId(); blockScanner.disableBlockPoolId(bpId); if (data != null) { data.shutdownBlockPool(bpId); } if (storage != null) { storage.removeBlockPoolStorage(bpId); } } }
blockScanner.addVolumeScanner(ref); } else {
volumeList.toArray(new FsVolumeImpl[volumeList.size()]))) { if (blockScanner != null) { blockScanner.removeVolumeScanner(target);
blockScanner.enableBlockPoolId(bpos.getBlockPoolId()); initDirectoryScanner(conf);
/** * Mark a block as "suspect." * * This means that we should try to rescan it soon. Note that the * VolumeScanner keeps a list of recently suspicious blocks, which * it uses to avoid rescanning the same block over and over in a short * time frame. * * @param storageId The ID of the storage where the block replica * is being stored. * @param block The block's ID and block pool id. */ synchronized void markSuspectBlock(String storageId, ExtendedBlock block) { if (!isEnabled()) { LOG.debug("Not scanning suspicious block {} on {}, because the block " + "scanner is disabled.", block, storageId); return; } VolumeScanner scanner = scanners.get(storageId); if (scanner == null) { // This could happen if the volume is in the process of being removed. // The removal process shuts down the VolumeScanner, but the volume // object stays around as long as there are references to it (which // should not be that long.) LOG.info("Not scanning suspicious block {} on {}, because there is no " + "volume scanner for that storageId.", block, storageId); return; } scanner.markSuspectBlock(block); }
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { response.setContentType("text/plain"); DataNode datanode = (DataNode) getServletContext().getAttribute("datanode"); BlockScanner blockScanner = datanode.getBlockScanner(); StringBuilder buffer = new StringBuilder(8 * 1024); if (!blockScanner.isEnabled()) { LOG.warn("Periodic block scanner is not running"); buffer.append("Periodic block scanner is not running. " + "Please check the datanode log if this is unexpected."); } else { buffer.append("Block Scanner Statistics\n\n"); blockScanner.printStats(buffer); } String resp = buffer.toString(); LOG.trace("Returned Servlet info {}", resp); response.getWriter().write(resp); } }
/** * Creates a dummy DataNode for testing purpose. */ @VisibleForTesting @InterfaceAudience.LimitedPrivate("HDFS") DataNode(final Configuration conf) throws DiskErrorException { super(conf); this.tracer = createTracer(conf); this.tracerConfigurationManager = new TracerConfigurationManager(DATANODE_HTRACE_PREFIX, conf); this.fileIoProvider = new FileIoProvider(conf, this); this.fileDescriptorPassingDisabledReason = null; this.maxNumberOfBlocksToLog = 0; this.confVersion = null; this.usersWithLocalPathAccess = null; this.connectToDnViaHostname = false; this.blockScanner = new BlockScanner(this, this.getConf()); this.pipelineSupportECN = false; this.socketFactory = NetUtils.getDefaultSocketFactory(conf); this.dnConf = new DNConf(this); initOOBTimeout(); storageLocationChecker = null; volumeChecker = new DatasetVolumeChecker(conf, new Timer()); }