/** * See {@link DirectoryScanner} */ private synchronized void initDirectoryScanner(Configuration conf) { if (directoryScanner != null) { return; } String reason = null; if (conf.getTimeDuration(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT, TimeUnit.SECONDS) < 0) { reason = "verification is turned off by configuration"; } else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) { reason = "verifcation is not supported by SimulatedFSDataset"; } if (reason == null) { directoryScanner = new DirectoryScanner(this, data, conf); directoryScanner.start(); } else { LOG.info("Periodic Directory Tree Verification scan " + "is disabled because {}", reason); } }
/** * Main program loop for DirectoryScanner. Runs {@link reconcile()} * and handles any exceptions. */ @Override public void run() { try { if (!shouldRun) { //shutdown has been activated LOG.warn("this cycle terminating immediately because 'shouldRun' has been deactivated"); return; } //We're are okay to run - do it reconcile(); } catch (Exception e) { //Log and continue - allows Executor to run again next cycle LOG.error("Exception during DirectoryScanner execution - will continue next cycle", e); } catch (Error er) { //Non-recoverable error - re-throw after logging the problem LOG.error("System Error during DirectoryScanner execution - permanently terminating periodic scanner", er); throw er; } }
private synchronized void shutdownDirectoryScanner() { if (directoryScanner != null) { directoryScanner.shutdown(); } }
/** * Reconcile differences between disk and in-memory blocks */ @VisibleForTesting public void reconcile() throws IOException { scan(); for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) { String bpid = entry.getKey(); LinkedList<ScanInfo> diff = entry.getValue(); for (ScanInfo info : diff) { dataset.checkAndUpdate(bpid, info); } } if (!retainDiffs) clear(); }
clear(); Map<String, ScanInfo[]> diskReport = getDiskReport(); addDifference(diffRecord, statsRecord, info); addDifference(diffRecord, statsRecord, memBlock.getBlockId(), info.getVolume()); m++; info.getBlockFile() == null) { addDifference(diffRecord, statsRecord, info); } else if (info.getGenStamp() != memBlock.getGenerationStamp() || info.getBlockLength() != memBlock.getNumBytes()) { addDifference(diffRecord, statsRecord, info); } else if (memBlock.compareWith(info) != 0) { addDifference(diffRecord, statsRecord, info); addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume()); if (!dataset.isDeletingBlock(bpid, blockpoolReport[d].getBlockId())) { statsRecord.missingMemoryBlocks++; addDifference(diffRecord, statsRecord, blockpoolReport[d]);
scanner = new DirectoryScanner(dataNode, fds, conf); ratio = runThrottleTest(blocks); retries -= 1; scanner = new DirectoryScanner(dataNode, fds, conf); ratio = runThrottleTest(blocks); retries -= 1; scanner = new DirectoryScanner(dataNode, fds, conf); ratio = runThrottleTest(blocks); retries -= 1; scanner = new DirectoryScanner(dataNode, fds, CONF); scanner.setRetainDiffs(true); scan(blocks, 0, 0, 0, 0, 0); scanner.shutdown(); assertFalse(scanner.getRunStatus()); scanner = new DirectoryScanner(dataNode, fds, conf); scanner.setRetainDiffs(true); scanner.reconcile(); assertFalse(scanner.getRunStatus()); LOG.info("Scanner took " + (Time.monotonicNow() - nowMs.get()) + "ms to shutdown"); DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 0); scanner = new DirectoryScanner(dataNode, fds, conf);
fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0)); client = cluster.getFileSystem().getClient(); scanner = new DirectoryScanner(dataNode, fds, CONF); scanner.setRetainDiffs(true); FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0)); scanner.shutdown(); scanner = null;
parallelism); DataNode dataNode = cluster.getDataNodes().get(0); scanner = new DirectoryScanner(dataNode, fds, CONF); scanner.setRetainDiffs(true); scanner.shutdown(); assertFalse(scanner.getRunStatus()); scanner.shutdown(); scanner = null;
private float runThrottleTest(int blocks) throws IOException { scanner.setRetainDiffs(true); scan(blocks, 0, 0, 0, 0, 0); scanner.shutdown(); assertFalse(scanner.getRunStatus()); return (float)scanner.timeWaitingMs.get() / scanner.timeRunningMs.get(); }
if (!retainDiffs) clear();
CONF.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, parallelism); scanner = new DirectoryScanner(fds, CONF);
clear(); Map<String, ScanInfo[]> diskReport = getDiskReport(); addDifference(diffRecord, statsRecord, info); addDifference(diffRecord, statsRecord, memBlock.getBlockId(), info.getVolume()); m++; addDifference(diffRecord, statsRecord, info); } else if (info.getGenStamp() != memBlock.getGenerationStamp() || info.getBlockFileLength() != memBlock.getNumBytes()) { addDifference(diffRecord, statsRecord, info); } else if (info.getBlockFile().compareTo(memBlock.getBlockFile()) != 0) { addDifference(diffRecord, statsRecord, info); addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume()); if (!dataset.isDeletingBlock(bpid, blockpoolReport[d].getBlockId())) { statsRecord.missingMemoryBlocks++; addDifference(diffRecord, statsRecord, blockpoolReport[d]);
fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0)); client = cluster.getFileSystem().getClient(); scanner = new DirectoryScanner(dataNode, fds, CONF); scanner.setRetainDiffs(true); FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0)); scanner.shutdown(); scanner = null;
/** * Reconcile differences between disk and in-memory blocks */ @VisibleForTesting void reconcile() throws IOException { scan(); for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) { String bpid = entry.getKey(); LinkedList<ScanInfo> diff = entry.getValue(); for (ScanInfo info : diff) { dataset.checkAndUpdate(bpid, info.getBlockId(), info.getBlockFile(), info.getMetaFile(), info.getVolume()); } } if (!retainDiffs) clear(); }
if (!retainDiffs) clear();
/** * See {@link DirectoryScanner} */ private synchronized void initDirectoryScanner(Configuration conf) { if (directoryScanner != null) { return; } String reason = null; if (conf.getInt(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT) < 0) { reason = "verification is turned off by configuration"; } else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) { reason = "verifcation is not supported by SimulatedFSDataset"; } if (reason == null) { directoryScanner = new DirectoryScanner(this, data, conf); directoryScanner.start(); } else { LOG.info("Periodic Directory Tree Verification scan is disabled because " + reason); } }
clear(); Map<String, ScanInfo[]> diskReport = getDiskReport(); addDifference(diffRecord, statsRecord, info); addDifference(diffRecord, statsRecord, memBlock.getBlockId(), info.getVolume()); m++; addDifference(diffRecord, statsRecord, info); } else if (info.getGenStamp() != memBlock.getGenerationStamp() || info.getBlockFileLength() != memBlock.getNumBytes()) { addDifference(diffRecord, statsRecord, info); } else if (info.getBlockFile().compareTo(memBlock.getBlockFile()) != 0) { addDifference(diffRecord, statsRecord, info); addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume()); if (!dataset.isDeletingBlock(bpid, blockpoolReport[d].getBlockId())) { statsRecord.missingMemoryBlocks++; addDifference(diffRecord, statsRecord, blockpoolReport[d]);
/** * Main program loop for DirectoryScanner. Runs {@link reconcile()} * and handles any exceptions. */ @Override public void run() { try { if (!shouldRun) { //shutdown has been activated LOG.warn("this cycle terminating immediately because 'shouldRun' has been deactivated"); return; } //We're are okay to run - do it reconcile(); } catch (Exception e) { //Log and continue - allows Executor to run again next cycle LOG.error("Exception during DirectoryScanner execution - will continue next cycle", e); } catch (Error er) { //Non-recoverable error - re-throw after logging the problem LOG.error("System Error during DirectoryScanner execution - permanently terminating periodic scanner", er); throw er; } }