Tabnine Logo
VolumeScanner
Code IndexAdd Tabnine to your IDE (free)

How to use
VolumeScanner
in
org.apache.hadoop.hdfs.server.datanode

Best Java code snippets using org.apache.hadoop.hdfs.server.datanode.VolumeScanner (Showing top 20 results out of 315)

origin: org.apache.hadoop/hadoop-hdfs

try {
 long monotonicMs = Time.monotonicNow();
 expireOldScannedBytesRecords(monotonicMs);
 if (!calculateShouldScan(volume.getStorageID(), conf.targetBytesPerSec,
   scannedBytesSum, startMinute, curMinute)) {
 } else {
  if ((curBlockIter == null) || curBlockIter.atEnd()) {
   long timeout = findNextUsableBlockIter();
   if (timeout > 0) {
    LOG.trace("{}: no block pools are ready to scan yet.  Waiting " +
   saveBlockIterator(curBlockIter);
   return 0;
   LOG.debug("{}: saving block iterator {} after {} ms.",
     this, curBlockIter, saveDelta);
   saveBlockIterator(curBlockIter);
 bytesScanned = scanBlock(block, conf.targetBytesPerSec);
 if (bytesScanned >= 0) {
  scannedBytesSum += bytesScanned;
origin: org.apache.hadoop/hadoop-hdfs

scanner = new VolumeScanner(conf, datanode, ref);
scanner.start();
scanners.put(volume.getStorageID(), scanner);
success = true;
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Disable scanning a given block pool id.
 *
 * @param bpid        The block pool id to disable scanning for.
 */
synchronized void disableBlockPoolId(String bpid) {
 Preconditions.checkNotNull(bpid);
 for (VolumeScanner scanner : scanners.values()) {
  scanner.disableBlockPoolId(bpid);
 }
}
origin: org.apache.hadoop/hadoop-hdfs

  suspectBlock = popNextSuspectBlock();
 timeout = runLoop(suspectBlock);
saveBlockIterator(iter);
IOUtils.cleanup(null, iter);
origin: ch.cern.hadoop/hadoop-hdfs

@Test(timeout=120000)
public void testCalculateNeededBytesPerSec() throws Exception {
 // If we didn't check anything the last hour, we should scan now.
 Assert.assertTrue(
   VolumeScanner.calculateShouldScan("test", 100, 0, 0, 60));
 // If, on average, we checked 101 bytes/s checked during the last hour,
 // stop checking now.
 Assert.assertFalse(VolumeScanner.
   calculateShouldScan("test", 100, 101 * 3600, 1000, 5000));
 // Target is 1 byte / s, but we didn't scan anything in the last minute.
 // Should scan now.
 Assert.assertTrue(VolumeScanner.
   calculateShouldScan("test", 1, 3540, 0, 60));
 // Target is 1000000 byte / s, but we didn't scan anything in the last
 // minute.  Should scan now.
 Assert.assertTrue(VolumeScanner.
   calculateShouldScan("test", 100000L, 354000000L, 0, 60));
 Assert.assertFalse(VolumeScanner.
   calculateShouldScan("test", 100000L, 365000000L, 0, 60));
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Enable scanning a given block pool id.
 *
 * @param bpid        The block pool id to enable scanning for.
 */
synchronized void enableBlockPoolId(String bpid) {
 Preconditions.checkNotNull(bpid);
 for (VolumeScanner scanner : scanners.values()) {
  scanner.enableBlockPoolId(bpid);
 }
}
origin: org.apache.hadoop/hadoop-hdfs

@VisibleForTesting
synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) {
 VolumeScanner scanner = scanners.get(volumeId);
 if (scanner == null) {
  return null;
 }
 return scanner.getStatistics();
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Shut down this scanner.
 */
public synchronized void shutdown() {
 stopping = true;
 notify();
 this.interrupt();
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Mark a block as "suspect."
 *
 * This means that we should try to rescan it soon.  Note that the
 * VolumeScanner keeps a list of recently suspicious blocks, which
 * it uses to avoid rescanning the same block over and over in a short
 * time frame.
 *
 * @param storageId     The ID of the storage where the block replica
 *                      is being stored.
 * @param block         The block's ID and block pool id.
 */
synchronized void markSuspectBlock(String storageId, ExtendedBlock block) {
 if (!isEnabled()) {
  LOG.debug("Not scanning suspicious block {} on {}, because the block " +
    "scanner is disabled.", block, storageId);
  return;
 }
 VolumeScanner scanner = scanners.get(storageId);
 if (scanner == null) {
  // This could happen if the volume is in the process of being removed.
  // The removal process shuts down the VolumeScanner, but the volume
  // object stays around as long as there are references to it (which
  // should not be that long.)
  LOG.info("Not scanning suspicious block {} on {}, because there is no " +
    "volume scanner for that storageId.", block, storageId);
  return;
 }
 scanner.markSuspectBlock(block);
}
origin: ch.cern.hadoop/hadoop-hdfs

  suspectBlock = popNextSuspectBlock();
 timeout = runLoop(suspectBlock);
saveBlockIterator(iter);
IOUtils.cleanup(null, iter);
origin: ch.cern.hadoop/hadoop-hdfs

/**
 * Enable scanning a given block pool id.
 *
 * @param bpid        The block pool id to enable scanning for.
 */
synchronized void enableBlockPoolId(String bpid) {
 Preconditions.checkNotNull(bpid);
 for (VolumeScanner scanner : scanners.values()) {
  scanner.enableBlockPoolId(bpid);
 }
}
origin: ch.cern.hadoop/hadoop-hdfs

@VisibleForTesting
synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) {
 VolumeScanner scanner = scanners.get(volumeId);
 if (scanner == null) {
  return null;
 }
 return scanner.getStatistics();
}
origin: ch.cern.hadoop/hadoop-hdfs

/**
 * Shut down this scanner.
 */
public synchronized void shutdown() {
 stopping = true;
 notify();
 this.interrupt();
}
origin: ch.cern.hadoop/hadoop-hdfs

/**
 * Mark a block as "suspect."
 *
 * This means that we should try to rescan it soon.  Note that the
 * VolumeScanner keeps a list of recently suspicious blocks, which
 * it uses to avoid rescanning the same block over and over in a short
 * time frame.
 *
 * @param storageId     The ID of the storage where the block replica
 *                      is being stored.
 * @param block         The block's ID and block pool id.
 */
synchronized void markSuspectBlock(String storageId, ExtendedBlock block) {
 if (!isEnabled()) {
  LOG.debug("Not scanning suspicious block {} on {}, because the block " +
    "scanner is disabled.", block, storageId);
  return;
 }
 VolumeScanner scanner = scanners.get(storageId);
 if (scanner == null) {
  // This could happen if the volume is in the process of being removed.
  // The removal process shuts down the VolumeScanner, but the volume
  // object stays around as long as there are references to it (which
  // should not be that long.)
  LOG.info("Not scanning suspicious block {} on {}, because there is no " +
    "volume scanner for that storageId.", block, storageId);
  return;
 }
 scanner.markSuspectBlock(block);
}
origin: ch.cern.hadoop/hadoop-hdfs

try {
 long monotonicMs = Time.monotonicNow();
 expireOldScannedBytesRecords(monotonicMs);
 if (!calculateShouldScan(volume.getStorageID(), conf.targetBytesPerSec,
   scannedBytesSum, startMinute, curMinute)) {
 } else {
  if ((curBlockIter == null) || curBlockIter.atEnd()) {
   long timeout = findNextUsableBlockIter();
   if (timeout > 0) {
    LOG.trace("{}: no block pools are ready to scan yet.  Waiting " +
   saveBlockIterator(curBlockIter);
   return 0;
   LOG.debug("{}: saving block iterator {} after {} ms.",
     this, curBlockIter, saveDelta);
   saveBlockIterator(curBlockIter);
 bytesScanned = scanBlock(block, conf.targetBytesPerSec);
 if (bytesScanned >= 0) {
  scannedBytesSum += bytesScanned;
origin: io.prestosql.hadoop/hadoop-apache

  suspectBlock = popNextSuspectBlock();
 timeout = runLoop(suspectBlock);
saveBlockIterator(iter);
IOUtils.cleanup(null, iter);
origin: ch.cern.hadoop/hadoop-hdfs

scanner = new VolumeScanner(conf, datanode, ref);
scanner.start();
scanners.put(volume.getStorageID(), scanner);
success = true;
origin: io.prestosql.hadoop/hadoop-apache

/**
 * Enable scanning a given block pool id.
 *
 * @param bpid        The block pool id to enable scanning for.
 */
synchronized void enableBlockPoolId(String bpid) {
 Preconditions.checkNotNull(bpid);
 for (VolumeScanner scanner : scanners.values()) {
  scanner.enableBlockPoolId(bpid);
 }
}
origin: ch.cern.hadoop/hadoop-hdfs

/**
 * Disable scanning a given block pool id.
 *
 * @param bpid        The block pool id to disable scanning for.
 */
synchronized void disableBlockPoolId(String bpid) {
 Preconditions.checkNotNull(bpid);
 for (VolumeScanner scanner : scanners.values()) {
  scanner.disableBlockPoolId(bpid);
 }
}
origin: io.prestosql.hadoop/hadoop-apache

@VisibleForTesting
synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) {
 VolumeScanner scanner = scanners.get(volumeId);
 if (scanner == null) {
  return null;
 }
 return scanner.getStatistics();
}
org.apache.hadoop.hdfs.server.datanodeVolumeScanner

Javadoc

VolumeScanner scans a single volume. Each VolumeScanner has its own thread.

They are all managed by the DataNode's BlockScanner.

Most used methods

  • calculateShouldScan
  • <init>
  • disableBlockPoolId
    Disallow the scanner from scanning the given block pool.
  • enableBlockPoolId
    Allow the scanner to scan the given block pool.
  • expireOldScannedBytesRecords
  • findNextUsableBlockIter
    Find a usable block iterator. We will consider available block iterators in order. This property is
  • getStatistics
  • interrupt
  • markSuspectBlock
  • popNextSuspectBlock
    If there are elements in the suspectBlocks list, removes and returns the first one. Otherwise, retur
  • positiveMsToHours
  • printStats
  • positiveMsToHours,
  • printStats,
  • runLoop,
  • saveBlockIterator,
  • scanBlock,
  • setConf,
  • setDaemon,
  • setName,
  • shutdown,
  • start

Popular in Java

  • Running tasks concurrently on multiple threads
  • addToBackStack (FragmentTransaction)
  • getSystemService (Context)
  • getSharedPreferences (Context)
  • BufferedImage (java.awt.image)
    The BufferedImage subclass describes an java.awt.Image with an accessible buffer of image data. All
  • URLEncoder (java.net)
    This class is used to encode a string using the format required by application/x-www-form-urlencoded
  • Timestamp (java.sql)
    A Java representation of the SQL TIMESTAMP type. It provides the capability of representing the SQL
  • TimeUnit (java.util.concurrent)
    A TimeUnit represents time durations at a given unit of granularity and provides utility methods to
  • AtomicInteger (java.util.concurrent.atomic)
    An int value that may be updated atomically. See the java.util.concurrent.atomic package specificati
  • IOUtils (org.apache.commons.io)
    General IO stream manipulation utilities. This class provides static utility methods for input/outpu
  • Github Copilot alternatives
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now