Tabnine Logo
FsDatasetAsyncDiskService
Code IndexAdd Tabnine to your IDE (free)

How to use
FsDatasetAsyncDiskService
in
org.apache.hadoop.hdfs.server.datanode.fsdataset.impl

Best Java code snippets using org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService (Showing top 20 results out of 315)

origin: org.apache.hadoop/hadoop-hdfs

public void submitSyncFileRangeRequest(FsVolumeImpl volume,
  final ReplicaOutputStreams streams, final long offset, final long nbytes,
  final int flags) {
 execute(volume, new Runnable() {
  @Override
  public void run() {
   try {
    streams.syncFileRangeIfPossible(offset, nbytes, flags);
   } catch (NativeIOException e) {
    LOG.warn("sync_file_range error", e);
   }
  }
 });
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Starts AsyncDiskService for a new volume
 * @param volume the root of the new data volume.
 */
synchronized void addVolume(FsVolumeImpl volume) {
 if (executors == null) {
  throw new RuntimeException("AsyncDiskService is already shutdown");
 }
 if (volume == null) {
  throw new RuntimeException("Attempt to add a null volume");
 }
 ThreadPoolExecutor executor = executors.get(volume.getStorageID());
 if (executor != null) {
  throw new RuntimeException("Volume " + volume + " is already existed.");
 }
 addExecutorForVolume(volume);
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Activate a volume to serve requests.
 * @throws IOException if the storage UUID already exists.
 */
private void activateVolume(
  ReplicaMap replicaMap,
  Storage.StorageDirectory sd, StorageType storageType,
  FsVolumeReference ref) throws IOException {
 try (AutoCloseableLock lock = datasetLock.acquire()) {
  DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid());
  if (dnStorage != null) {
   final String errorMsg = String.format(
     "Found duplicated storage UUID: %s in %s.",
     sd.getStorageUuid(), sd.getVersionFile());
   LOG.error(errorMsg);
   throw new IOException(errorMsg);
  }
  volumeMap.addAll(replicaMap);
  storageMap.put(sd.getStorageUuid(),
    new DatanodeStorage(sd.getStorageUuid(),
      DatanodeStorage.State.NORMAL,
      storageType));
  asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume());
  volumes.addVolume(ref);
 }
}
origin: org.apache.hadoop/hadoop-hdfs

 asyncDiskService.deleteAsync(v.obtainReference(), removing,
   new ExtendedBlock(bpid, invalidBlks[i]),
   dataStorage.getTrashDirectoryForReplica(bpid, removing));
} else {
 asyncDiskService.deleteSync(v.obtainReference(), removing,
   new ExtendedBlock(bpid, invalidBlks[i]),
   dataStorage.getTrashDirectoryForReplica(bpid, removing));
origin: org.apache.hadoop/hadoop-hdfs

@Override
public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block,
  ReplicaOutputStreams outs, long offset, long nbytes, int flags) {
 FsVolumeImpl fsVolumeImpl = this.getVolume(block);
 asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, outs, offset,
   nbytes, flags);
}
origin: org.apache.hadoop/hadoop-hdfs

@Override // FsDatasetSpi
public void shutdown() {
 fsRunning = false;
 if (lazyWriter != null) {
  ((LazyWriter) lazyWriter.getRunnable()).stop();
  lazyWriter.interrupt();
 }
 if (mbeanName != null) {
  MBeans.unregister(mbeanName);
 }
 
 if (asyncDiskService != null) {
  asyncDiskService.shutdown();
 }
 if (asyncLazyPersistService != null) {
  asyncLazyPersistService.shutdown();
 }
 
 if(volumes != null) {
  volumes.shutdown();
 }
 if (lazyWriter != null) {
  try {
   lazyWriter.join();
  } catch (InterruptedException ie) {
   LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " +
          "from LazyWriter.join");
  }
 }
}
origin: org.apache.hadoop/hadoop-hdfs

  sd.getStorageUuid() + " from FsDataset.");
asyncDiskService.removeVolume(sd.getStorageUuid());
volumes.removeVolume(sdLocation, clearFailure);
volumes.waitVolumeRemoved(5000, datasetLockCondition);
origin: org.apache.hadoop/hadoop-hdfs

volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(),
  blockChooserImpl);
asyncDiskService = new FsDatasetAsyncDiskService(datanode, this);
asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode, conf);
deletingBlock = new HashMap<String, Set<Long>>();
origin: ch.cern.hadoop/hadoop-hdfs

 asyncDiskService.deleteAsync(v.obtainReference(), f,
   FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()),
   new ExtendedBlock(bpid, invalidBlks[i]),
   dataStorage.getTrashDirectoryForBlockFile(bpid, f));
} else {
 asyncDiskService.deleteSync(v.obtainReference(), f,
   FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()),
   new ExtendedBlock(bpid, invalidBlks[i]),
origin: ch.cern.hadoop/hadoop-hdfs

@Override
public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block,
  FileDescriptor fd, long offset, long nbytes, int flags) {
 FsVolumeImpl fsVolumeImpl = this.getVolume(block);
 asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, fd, offset,
   nbytes, flags);
}
origin: ch.cern.hadoop/hadoop-hdfs

@Override // FsDatasetSpi
public void shutdown() {
 fsRunning = false;
 ((LazyWriter) lazyWriter.getRunnable()).stop();
 lazyWriter.interrupt();
 if (mbeanName != null) {
  MBeans.unregister(mbeanName);
 }
 
 if (asyncDiskService != null) {
  asyncDiskService.shutdown();
 }
 if (asyncLazyPersistService != null) {
  asyncLazyPersistService.shutdown();
 }
 
 if(volumes != null) {
  volumes.shutdown();
 }
 try {
  lazyWriter.join();
 } catch (InterruptedException ie) {
  LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " +
       "from LazyWriter.join");
 }
}
origin: ch.cern.hadoop/hadoop-hdfs

asyncDiskService.removeVolume(sd.getCurrentDir());
volumes.removeVolume(absRoot, clearFailure);
origin: ch.cern.hadoop/hadoop-hdfs

volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(),
  blockChooserImpl);
asyncDiskService = new FsDatasetAsyncDiskService(datanode, this);
asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode);
deletingBlock = new HashMap<String, Set<Long>>();
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Delete the block file and meta file from the disk asynchronously, adjust
 * dfsUsed statistics accordingly.
 */
void deleteAsync(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete,
  ExtendedBlock block, String trashDirectory) {
 LOG.info("Scheduling " + block.getLocalBlock()
   + " replica " + replicaToDelete + " for deletion");
 ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask(
   volumeRef, replicaToDelete, block, trashDirectory);
 execute(((FsVolumeImpl) volumeRef.getVolume()), deletionTask);
}
origin: io.prestosql.hadoop/hadoop-apache

 asyncDiskService.deleteAsync(v.obtainReference(), f,
   FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()),
   new ExtendedBlock(bpid, invalidBlks[i]),
   dataStorage.getTrashDirectoryForBlockFile(bpid, f));
} else {
 asyncDiskService.deleteSync(v.obtainReference(), f,
   FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()),
   new ExtendedBlock(bpid, invalidBlks[i]),
origin: ch.cern.hadoop/hadoop-hdfs

private void addVolume(Collection<StorageLocation> dataLocations,
  Storage.StorageDirectory sd) throws IOException {
 final File dir = sd.getCurrentDir();
 final StorageType storageType =
   getStorageTypeFromLocations(dataLocations, sd.getRoot());
 // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is
 // nothing needed to be rolled back to make various data structures, e.g.,
 // storageMap and asyncDiskService, consistent.
 FsVolumeImpl fsVolume = new FsVolumeImpl(
   this, sd.getStorageUuid(), dir, this.conf, storageType);
 FsVolumeReference ref = fsVolume.obtainReference();
 ReplicaMap tempVolumeMap = new ReplicaMap(this);
 fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker);
 synchronized (this) {
  volumeMap.addAll(tempVolumeMap);
  storageMap.put(sd.getStorageUuid(),
    new DatanodeStorage(sd.getStorageUuid(),
      DatanodeStorage.State.NORMAL,
      storageType));
  asyncDiskService.addVolume(sd.getCurrentDir());
  volumes.addVolume(ref);
 }
 LOG.info("Added volume - " + dir + ", StorageType: " + storageType);
}
origin: io.prestosql.hadoop/hadoop-apache

@Override
public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block,
  FileDescriptor fd, long offset, long nbytes, int flags) {
 FsVolumeImpl fsVolumeImpl = this.getVolume(block);
 asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, fd, offset,
   nbytes, flags);
}
origin: ch.cern.hadoop/hadoop-hdfs

/**
 * Starts AsyncDiskService for a new volume
 * @param volume the root of the new data volume.
 */
synchronized void addVolume(File volume) {
 if (executors == null) {
  throw new RuntimeException("AsyncDiskService is already shutdown");
 }
 ThreadPoolExecutor executor = executors.get(volume);
 if (executor != null) {
  throw new RuntimeException("Volume " + volume + " is already existed.");
 }
 addExecutorForVolume(volume);
}
origin: io.prestosql.hadoop/hadoop-apache

@Override // FsDatasetSpi
public void shutdown() {
 fsRunning = false;
 ((LazyWriter) lazyWriter.getRunnable()).stop();
 lazyWriter.interrupt();
 if (mbeanName != null) {
  MBeans.unregister(mbeanName);
 }
 
 if (asyncDiskService != null) {
  asyncDiskService.shutdown();
 }
 if (asyncLazyPersistService != null) {
  asyncLazyPersistService.shutdown();
 }
 
 if(volumes != null) {
  volumes.shutdown();
 }
 try {
  lazyWriter.join();
 } catch (InterruptedException ie) {
  LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " +
       "from LazyWriter.join");
 }
}
origin: io.prestosql.hadoop/hadoop-apache

asyncDiskService.removeVolume(sd.getCurrentDir());
volumes.removeVolume(absRoot, clearFailure);
org.apache.hadoop.hdfs.server.datanode.fsdataset.implFsDatasetAsyncDiskService

Javadoc

This class is a container of multiple thread pools, each for a volume, so that we can schedule async disk operations easily. Examples of async disk operations are deletion of block files. We don't want to create a new thread for each of the deletion request, and we don't want to do all deletions in the heartbeat thread since deletion can be slow, and we don't want to use a single thread pool because that is inefficient when we have more than 1 volume. AsyncDiskService is the solution for these. Another example of async disk operation is requesting sync_file_range(). This class and org.apache.hadoop.util.AsyncDiskService are similar. They should be combined.

Most used methods

  • execute
    Execute the task sometime in the future, using ThreadPools.
  • <init>
    Create a AsyncDiskServices with a set of volumes (specified by their root directories). The AsyncDis
  • addExecutorForVolume
  • addVolume
    Starts AsyncDiskService for a new volume
  • deleteAsync
    Delete the block file and meta file from the disk asynchronously, adjust dfsUsed statistics accordin
  • deleteSync
    Delete the block file and meta file from the disk synchronously, adjust dfsUsed statistics according
  • removeVolume
    Stops AsyncDiskService for a volume.
  • shutdown
    Gracefully shut down all ThreadPool. Will wait for all deletion tasks to finish.
  • submitSyncFileRangeRequest
  • countPendingDeletions

Popular in Java

  • Making http post requests using okhttp
  • getSystemService (Context)
  • getApplicationContext (Context)
  • runOnUiThread (Activity)
  • Socket (java.net)
    Provides a client-side TCP socket.
  • Map (java.util)
    A Map is a data structure consisting of a set of keys and values in which each key is mapped to a si
  • Queue (java.util)
    A collection designed for holding elements prior to processing. Besides basic java.util.Collection o
  • TreeSet (java.util)
    TreeSet is an implementation of SortedSet. All optional operations (adding and removing) are support
  • ExecutorService (java.util.concurrent)
    An Executor that provides methods to manage termination and methods that can produce a Future for tr
  • LoggerFactory (org.slf4j)
    The LoggerFactory is a utility class producing Loggers for various logging APIs, most notably for lo
  • Top plugins for Android Studio
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now