congrats Icon
New! Tabnine Pro 14-day free trial
Start a free trial
Tabnine Logo
org.apache.hadoop.hdfs.server.blockmanagement
Code IndexAdd Tabnine to your IDE (free)

How to use org.apache.hadoop.hdfs.server.blockmanagement

Best Java code snippets using org.apache.hadoop.hdfs.server.blockmanagement (Showing top 20 results out of 315)

origin: org.apache.hadoop/hadoop-hdfs

public void removeBlockFromMap(BlockInfo block) {
 for(DatanodeStorageInfo info : blocksMap.getStorages(block)) {
  excessRedundancyMap.remove(info.getDatanodeDescriptor(), block);
 }
 blocksMap.removeBlock(block);
 // If block is removed from blocksMap remove it from corruptReplicasMap
 corruptReplicas.removeFromCorruptReplicasMap(block);
}
origin: org.apache.hadoop/hadoop-hdfs

public void clear() {
 blockIdManager.clear();
 clearQueues();
 blocksMap.clear();
}
origin: org.apache.hadoop/hadoop-hdfs

public static LocatedBlock newLocatedBlock(ExtendedBlock eb, BlockInfo info,
  DatanodeStorageInfo[] locs, long offset) throws IOException {
 final LocatedBlock lb;
 if (info.isStriped()) {
  lb = newLocatedStripedBlock(eb, locs,
    info.getUnderConstructionFeature().getBlockIndices(),
    offset, false);
 } else {
  lb = newLocatedBlock(eb, locs, offset, false);
 }
 return lb;
}
origin: org.apache.hadoop/hadoop-hdfs

@Override
void chooseTargets(BlockPlacementPolicy blockplacement,
  BlockStoragePolicySuite storagePolicySuite,
  Set<Node> excludedNodes) {
 // TODO: new placement policy for EC considering multiple writers
 DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
   getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0],
   getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
   storagePolicySuite.getPolicy(getStoragePolicyID()), null);
 setTargets(chosenTargets);
}
origin: org.apache.hadoop/hadoop-hdfs

void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock,
  DatanodeStorageInfo storageInfo) throws IOException {
 BlockInfo block = ucBlock.storedBlock;
 block.getUnderConstructionFeature().addReplicaIfNotPresent(
   storageInfo, ucBlock.reportedBlock, ucBlock.reportedState);
 // Add replica if appropriate. If the replica was previously corrupt
 // but now okay, it might need to be updated.
 if (ucBlock.reportedState == ReplicaState.FINALIZED && (
   block.findStorageInfo(storageInfo) < 0) || corruptReplicas
   .isReplicaCorrupt(block, storageInfo.getDatanodeDescriptor())) {
  addStoredBlock(block, ucBlock.reportedBlock, storageInfo, null, true);
 }
} 
origin: org.apache.hadoop/hadoop-hdfs

/** Increment the number of blocks scheduled for each given storage */ 
public static void incrementBlocksScheduled(DatanodeStorageInfo... storages) {
 for (DatanodeStorageInfo s : storages) {
  s.getDatanodeDescriptor().incrementBlocksScheduled(s.getStorageType());
 }
}
origin: org.apache.hadoop/hadoop-hdfs

boolean isNeededReconstructionForMaintenance(BlockInfo storedBlock,
  NumberReplicas numberReplicas) {
 return storedBlock.isComplete() && (numberReplicas.liveReplicas() <
   getMinMaintenanceStorageNum(storedBlock) ||
   !isPlacementPolicySatisfied(storedBlock));
}
origin: org.apache.hadoop/hadoop-hdfs

boolean hasEnoughEffectiveReplicas(BlockInfo block,
  NumberReplicas numReplicas, int pendingReplicaNum) {
 int required = getExpectedLiveRedundancyNum(block, numReplicas);
 int numEffectiveReplicas = numReplicas.liveReplicas() + pendingReplicaNum;
 return (numEffectiveReplicas >= required) &&
   (pendingReplicaNum > 0 || isPlacementPolicySatisfied(block));
}
origin: org.apache.hadoop/hadoop-hdfs

 public boolean hasStorageType(StorageType type) {
  for (DatanodeStorageInfo dnStorage : getStorageInfos()) {
   if (dnStorage.getStorageType() == type) {
    return true;
   }
  }
  return false;
 }
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Force the given block in the given file to be marked as complete,
 * regardless of whether enough replicas are present. This is necessary
 * when tailing edit logs as a Standby.
 */
public void forceCompleteBlock(final BlockInfo block) throws IOException {
 List<ReplicaUnderConstruction> staleReplicas = block.commitBlock(block);
 removeStaleReplicas(staleReplicas, block);
 completeBlock(block, null, true);
}
origin: org.apache.hadoop/hadoop-hdfs

void updateState() {
 pendingReconstructionBlocksCount = pendingReconstruction.size();
 lowRedundancyBlocksCount = neededReconstruction.size();
 corruptReplicaBlocksCount = corruptReplicas.size();
}
origin: org.apache.hadoop/hadoop-hdfs

@Override
boolean removeBlock(BlockInfo b) {
 ProvidedDescriptor dn = (ProvidedDescriptor) getDatanodeDescriptor();
 if (dn.activeProvidedDatanodes() == 0) {
  return super.removeBlock(b);
 } else {
  return false;
 }
}
origin: org.apache.hadoop/hadoop-hdfs

public DatanodeStorageInfo[] getStorages(BlockInfo block) {
 final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()];
 int i = 0;
 for(DatanodeStorageInfo s : blocksMap.getStorages(block)) {
  storages[i++] = s;
 }
 return storages;
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Is data-node the replica belongs to alive.
 */
boolean isAlive() {
 return expectedLocation.getDatanodeDescriptor().isAlive();
}
origin: org.apache.hadoop/hadoop-hdfs

public BlockIdManager(BlockManager blockManager) {
 this.legacyGenerationStampLimit =
   HdfsConstants.GRANDFATHER_GENERATION_STAMP;
 this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager);
 this.blockGroupIdGenerator = new SequentialBlockGroupIdGenerator(blockManager);
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Adds block to list of blocks which will be invalidated on specified
 * datanode and log the operation
 */
void addToInvalidates(final Block block, final DatanodeInfo datanode) {
 if (!isPopulatingReplQueues()) {
  return;
 }
 invalidateBlocks.add(block, datanode, true);
}
origin: org.apache.hadoop/hadoop-hdfs

public short getExpectedRedundancyNum(BlockInfo block) {
 return block.isStriped() ?
   ((BlockInfoStriped) block).getRealTotalBlockNum() :
   block.getReplication();
}
origin: org.apache.hadoop/hadoop-hdfs

@Override
public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
  int numberOfReplicas) {
 BlockPlacementStatus defaultStatus = super.verifyBlockPlacement(locs,
   numberOfReplicas);
 BlockPlacementStatusWithUpgradeDomain upgradeDomainStatus =
   new BlockPlacementStatusWithUpgradeDomain(defaultStatus,
     getUpgradeDomainsFromNodes(locs),
       numberOfReplicas, upgradeDomainFactor);
 return upgradeDomainStatus;
}
origin: org.apache.hadoop/hadoop-hdfs

private Block getBlockOnStorage(BlockInfo storedBlock,
  DatanodeStorageInfo storage) {
 return storedBlock.isStriped() ?
   ((BlockInfoStriped) storedBlock).getBlockOnStorage(storage) : storedBlock;
}
origin: org.apache.hadoop/hadoop-hdfs

/**
 * Create array of expected replica locations
 * (as has been assigned by chooseTargets()).
 */
public DatanodeStorageInfo[] getExpectedStorageLocations() {
 int numLocations = getNumExpectedLocations();
 DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations];
 for (int i = 0; i < numLocations; i++) {
  storages[i] = replicas[i].getExpectedStorageLocation();
 }
 return storages;
}
org.apache.hadoop.hdfs.server.blockmanagement

Most used classes

  • AvailableSpaceBlockPlacementPolicy
    Space balanced block placement policy.
  • BlockCollection
    This interface is used by the block manager to expose a few characteristics of a collection of Block
  • BlockIdManager
    BlockIdManager allocates the generation stamps and the block ID. The FSNamesystem is responsible for
  • BlockInfoContiguous
    BlockInfo class maintains for a given block the INodeFile it is part of and datanodes where the repl
  • BlockManager
    Keeps information related to the blocks stored in the Hadoop cluster.
  • BlockPlacementPolicyDefault,
  • BlockStoragePolicySuite,
  • BlocksMap,
  • CorruptReplicasMap,
  • DatanodeDescriptor$CachedBlocksList,
  • DatanodeDescriptor,
  • DatanodeManager,
  • DatanodeStorageInfo,
  • FSClusterStats,
  • HeartbeatManager,
  • Host2NodesMap,
  • HostFileManager,
  • InvalidateBlocks,
  • NumberReplicas
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now