private LocalReplica buildRUR() throws IllegalArgumentException { if (null == fromReplica) { throw new IllegalArgumentException( "Missing a valid replica to recover from"); } if (null != writer || null != block) { throw new IllegalArgumentException("Invalid state for " + "recovering from replica with blk id " + fromReplica.getBlockId()); } if (fromReplica.getState() == ReplicaState.RUR) { return new ReplicaUnderRecovery((ReplicaUnderRecovery) fromReplica); } else { return new ReplicaUnderRecovery(fromReplica, recoveryId); } }
/** * Invalidate a block but does not delete the actual on-disk block file. * * It should only be used when deactivating disks. * * @param bpid the block pool ID. * @param block The block to be invalidated. */ public void invalidate(String bpid, ReplicaInfo block) { // If a DFSClient has the replica in its cache of short-circuit file // descriptors (and the client is using ShortCircuitShm), invalidate it. datanode.getShortCircuitRegistry().processBlockInvalidation( new ExtendedBlockId(block.getBlockId(), bpid)); // If the block is cached, start uncaching it. cacheManager.uncacheBlock(bpid, block.getBlockId()); datanode.notifyNamenodeDeletedBlock(new ExtendedBlock(bpid, block), block.getStorageUuid()); }
/** * Add a replica's meta information into the map, if already exist * return the old replicaInfo. */ ReplicaInfo addAndGet(String bpid, ReplicaInfo replicaInfo) { checkBlockPool(bpid); checkBlock(replicaInfo); try (AutoCloseableLock l = lock.acquire()) { FoldedTreeSet<ReplicaInfo> set = map.get(bpid); if (set == null) { // Add an entry for block pool if it does not exist already set = new FoldedTreeSet<>(); map.put(bpid, set); } ReplicaInfo oldReplicaInfo = set.get(replicaInfo.getBlockId(), LONG_AND_BLOCK_COMPARATOR); if (oldReplicaInfo != null) { return oldReplicaInfo; } else { set.addOrReplace(replicaInfo); } return replicaInfo; } }
final StorageLocation blockStorageLocation = block.getVolume().getStorageLocation(); LOG.trace("checking for block " + block.getBlockId() + " with storageLocation " + blockStorageLocation); if (blockStorageLocation.equals(sdLocation)) {
/** * Move a persisted replica from lazypersist directory to a subdirectory * under finalized. */ ReplicaInfo activateSavedReplica(ReplicaInfo replicaInfo, RamDiskReplica replicaState) throws IOException { File metaFile = replicaState.getSavedMetaFile(); File blockFile = replicaState.getSavedBlockFile(); final long blockId = replicaInfo.getBlockId(); final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId); final File targetBlockFile = new File(blockDir, blockFile.getName()); final File targetMetaFile = new File(blockDir, metaFile.getName()); fileIoProvider.moveFile(volume, blockFile, targetBlockFile); FsDatasetImpl.LOG.info("Moved " + blockFile + " to " + targetBlockFile); fileIoProvider.moveFile(volume, metaFile, targetMetaFile); FsDatasetImpl.LOG.info("Moved " + metaFile + " to " + targetMetaFile); ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.FINALIZED) .setBlockId(blockId) .setLength(replicaInfo.getBytesOnDisk()) .setGenerationStamp(replicaInfo.getGenerationStamp()) .setFsVolume(replicaState.getLazyPersistVolume()) .setDirectoryToUse(targetBlockFile.getParentFile()) .build(); return newReplicaInfo; }
+ "Replica with blk id " + replicaInfo.getBlockId() + " has state " + replicaInfo.getState()); cacheManager.uncacheBlock(bpid, replicaInfo.getBlockId()); newGS, estimateBlockLen); if (rip.getReplicaInfo().getState() != ReplicaState.RBW) { throw new IOException("Append on block " + replicaInfo.getBlockId() + " returned a replica of state " + rip.getReplicaInfo().getState() + "; expected RBW");
+ oldReplicaInfo.getBlockId() + " should be derived from LocalReplica");
addDeletingBlock(bpid, removing.getBlockId()); if (LOG.isDebugEnabled()) { LOG.debug("Block file " + removing.getBlockURI()
ReplicaInfo memBlock = bl.get(m); ScanInfo info = blockpoolReport[d]; if (info.getBlockId() < memBlock.getBlockId()) { if (!dataset.isDeletingBlock(bpid, info.getBlockId())) { continue; if (info.getBlockId() > memBlock.getBlockId()) { memBlock.getBlockId(), info.getVolume()); m++; continue; ReplicaInfo current = bl.get(m++); addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume());
boolean copyOnTruncate = newBlockId > 0L && rur.getBlockId() != newBlockId; rur, bpid, newBlockId, recoveryId, newlength); if (newReplicaInfo.getState() != ReplicaState.RBW) { throw new IOException("Append on block " + rur.getBlockId() + " returned a replica of state " + newReplicaInfo.getState() + "; expected RBW");
public ReplicaInfo moveBlockToTmpLocation(ExtendedBlock block, ReplicaInfo replicaInfo, int smallBufferSize, Configuration conf) throws IOException { File[] blockFiles = FsDatasetImpl.copyBlockFiles(block.getBlockId(), block.getGenerationStamp(), replicaInfo, getTmpDir(block.getBlockPoolId()), replicaInfo.isOnTransientStorage(), smallBufferSize, conf); ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.TEMPORARY) .setBlockId(replicaInfo.getBlockId()) .setGenerationStamp(replicaInfo.getGenerationStamp()) .setFsVolume(this) .setDirectoryToUse(blockFiles[0].getParentFile()) .setBytesToReserve(0) .build(); newReplicaInfo.setNumBytes(blockFiles[1].length()); return newReplicaInfo; }
try (AutoCloseableLock lock = datasetLock.acquire()) { if (volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp() > replicaInfo.getGenerationStamp()) { throw new IOException("Generation Stamp should be monotonically " false); ramDiskReplicaTracker.addReplica( bpid, replicaInfo.getBlockId(), v, replicaInfo.getNumBytes()); datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes());
assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == recoveryId && finalized.getNumBytes() == newlength + ", newBlockId=" + newBlockId + ", finalized=" + finalized; } else { assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == oldBlock.getGenerationStamp() && finalized.getNumBytes() == oldBlock.getNumBytes()
.setBlockId(replicaInfo.getBlockId()) .setLength(replicaInfo.getNumBytes()) .setGenerationStamp(newGS)
.build(); ReplicaInfo oldReplica = volumeMap.get(bpid, newReplica.getBlockId()); if (oldReplica == null) { volumeMap.add(bpid, newReplica); incDfsUsed(region.getBlock().getNumBytes()); } else { LOG.warn("A block with id " + newReplica.getBlockId() + " exists locally. Skipping PROVIDED replica");
/** * Return the parent directory path where this replica is located * @return the parent directory path where this replica is located */ File getDir() { return hasSubdirs ? DatanodeUtil.idToBlockDir(baseDir, getBlockId()) : baseDir; }
/** * Return the parent directory path where this replica is located * @return the parent directory path where this replica is located */ File getDir() { return hasSubdirs ? DatanodeUtil.idToBlockDir(baseDir, getBlockId()) : baseDir; }
public ReplicaRecoveryInfo createInfo() { return new ReplicaRecoveryInfo(original.getBlockId(), original.getBytesOnDisk(), original.getGenerationStamp(), original.getState()); } }
public ReplicaRecoveryInfo createInfo() { return new ReplicaRecoveryInfo(original.getBlockId(), original.getBytesOnDisk(), original.getGenerationStamp(), original.getState()); } }
private static void assertEquals(ReplicaInfo originalInfo, ReplicaRecoveryInfo recoveryInfo) { Assert.assertEquals(originalInfo.getBlockId(), recoveryInfo.getBlockId()); Assert.assertEquals(originalInfo.getGenerationStamp(), recoveryInfo.getGenerationStamp()); Assert.assertEquals(originalInfo.getBytesOnDisk(), recoveryInfo.getNumBytes()); Assert.assertEquals(originalInfo.getState(), recoveryInfo.getOriginalReplicaState()); }