public ReplicaInfo moveBlockToTmpLocation(ExtendedBlock block, ReplicaInfo replicaInfo, int smallBufferSize, Configuration conf) throws IOException { File[] blockFiles = FsDatasetImpl.copyBlockFiles(block.getBlockId(), block.getGenerationStamp(), replicaInfo, getTmpDir(block.getBlockPoolId()), replicaInfo.isOnTransientStorage(), smallBufferSize, conf); ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.TEMPORARY) .setBlockId(replicaInfo.getBlockId()) .setGenerationStamp(replicaInfo.getGenerationStamp()) .setFsVolume(this) .setDirectoryToUse(blockFiles[0].getParentFile()) .setBytesToReserve(0) .build(); newReplicaInfo.setNumBytes(blockFiles[1].length()); return newReplicaInfo; }
/** * Cleanup the old replica and notifies the NN about new replica. * * @param replicaInfo - Old replica to be deleted * @param newReplicaInfo - New replica object * @param bpid - block pool id */ private void removeOldReplica(ReplicaInfo replicaInfo, ReplicaInfo newReplicaInfo, final String bpid) { // Before deleting the files from old storage we must notify the // NN that the files are on the new storage. Else a blockReport from // the transient storage might cause the NN to think the blocks are lost. // Replicas must be evicted from client short-circuit caches, because the // storage will no longer be same, and thus will require validating // checksum. This also stops a client from holding file descriptors, // which would prevent the OS from reclaiming the memory. ExtendedBlock extendedBlock = new ExtendedBlock(bpid, newReplicaInfo); datanode.getShortCircuitRegistry().processBlockInvalidation( ExtendedBlockId.fromExtendedBlock(extendedBlock)); datanode.notifyNamenodeReceivedBlock( extendedBlock, null, newReplicaInfo.getStorageUuid(), newReplicaInfo.isOnTransientStorage()); // Remove the old replicas cleanupReplica(bpid, replicaInfo); // If deletion failed then the directory scanner will cleanup the blocks // eventually. }
if (replicaInfo.isOnTransientStorage()) {
private void removeOldReplica(ReplicaInfo replicaInfo, ReplicaInfo newReplicaInfo, File blockFile, File metaFile, long blockFileUsed, long metaFileUsed, final String bpid) { // Before deleting the files from old storage we must notify the // NN that the files are on the new storage. Else a blockReport from // the transient storage might cause the NN to think the blocks are lost. // Replicas must be evicted from client short-circuit caches, because the // storage will no longer be same, and thus will require validating // checksum. This also stops a client from holding file descriptors, // which would prevent the OS from reclaiming the memory. ExtendedBlock extendedBlock = new ExtendedBlock(bpid, newReplicaInfo); datanode.getShortCircuitRegistry().processBlockInvalidation( ExtendedBlockId.fromExtendedBlock(extendedBlock)); datanode.notifyNamenodeReceivedBlock( extendedBlock, null, newReplicaInfo.getStorageUuid(), newReplicaInfo.isOnTransientStorage()); // Remove the old replicas if (blockFile.delete() || !blockFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed); if (metaFile.delete() || !metaFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed); } } // If deletion failed then the directory scanner will cleanup the blocks // eventually. }
private void removeOldReplica(ReplicaInfo replicaInfo, ReplicaInfo newReplicaInfo, File blockFile, File metaFile, long blockFileUsed, long metaFileUsed, final String bpid) { // Before deleting the files from old storage we must notify the // NN that the files are on the new storage. Else a blockReport from // the transient storage might cause the NN to think the blocks are lost. // Replicas must be evicted from client short-circuit caches, because the // storage will no longer be same, and thus will require validating // checksum. This also stops a client from holding file descriptors, // which would prevent the OS from reclaiming the memory. ExtendedBlock extendedBlock = new ExtendedBlock(bpid, newReplicaInfo); datanode.getShortCircuitRegistry().processBlockInvalidation( ExtendedBlockId.fromExtendedBlock(extendedBlock)); datanode.notifyNamenodeReceivedBlock( extendedBlock, null, newReplicaInfo.getStorageUuid(), newReplicaInfo.isOnTransientStorage()); // Remove the old replicas if (blockFile.delete() || !blockFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed); if (metaFile.delete() || !metaFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed); } } // If deletion failed then the directory scanner will cleanup the blocks // eventually. }
if (replicaInfo.isOnTransientStorage()) { block.getGenerationStamp(), oldMetaFile, oldBlockFile, targetVolume.getTmpDir(block.getBlockPoolId()), replicaInfo.isOnTransientStorage());
if (replicaInfo.isOnTransientStorage()) { block.getGenerationStamp(), oldMetaFile, oldBlockFile, targetVolume.getTmpDir(block.getBlockPoolId()), replicaInfo.isOnTransientStorage());