/** * Return the parent directory path where this replica is located. * @return the parent directory path where this replica is located */ protected File getDir() { return hasSubdirs ? DatanodeUtil.idToBlockDir(baseDir, getBlockId()) : baseDir; }
/** * Verify whether the actual directory location of block file has the * expected directory path computed using its block ID. */ private void verifyFileLocation(File actualBlockFile, File bpFinalizedDir, long blockId) { File expectedBlockDir = DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId); File actualBlockDir = actualBlockFile.getParentFile(); if (actualBlockDir.compareTo(expectedBlockDir) != 0) { LOG.warn("Block: " + blockId + " found in invalid directory. Expected directory: " + expectedBlockDir + ". Actual directory: " + actualBlockDir); } }
private File getBlockFile(String bpid, ExtendedBlock blk) throws IOException { return new File(DatanodeUtil.idToBlockDir(getFinalizedDir(bpid), blk.getBlockId()).toString() + "/" + blk.getBlockName()); }
/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, ReplicaInfo srcReplica, File destRoot, boolean calculateChecksum, int smallBufferSize, final Configuration conf) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); // blockName is same as the filename for the block final File dstFile = new File(destDir, srcReplica.getBlockName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcReplica, dstMeta, dstFile, calculateChecksum, smallBufferSize, conf); }
for (String blockName : blockNames) { long blockId = Block.getBlockId(blockName); File blockLocation = DatanodeUtil.idToBlockDir(blockRoot, blockId); if (!blockLocation.exists()) { if (!blockLocation.mkdirs()) {
File blockFile = Block.metaToBlockFile(metaFile); long blockId = Block.filename2id(blockFile.getName()); File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaInfo replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return FsDatasetImpl.copyBlockFiles(replicaInfo, dstMetaFile, dstBlockFile, true, DFSUtilClient.getSmallBufferSize(conf), conf); }
File addFinalizedBlock(Block b, ReplicaInfo replicaInfo) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); fileIoProvider.mkdirsWithExistsCheck(volume, blockDir); File blockFile = FsDatasetImpl.moveBlockFiles(b, replicaInfo, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); if (dfsUsage instanceof CachingGetSpaceUsed) { ((CachingGetSpaceUsed) dfsUsage).incDfsUsed( b.getNumBytes() + metaFile.length()); } return blockFile; }
ExtendedBlock block = new ExtendedBlock(bpid, Block.filename2id(state.curEntry)); File expectedBlockDir = DatanodeUtil.idToBlockDir( new File("."), block.getBlockId()); File actualBlockDir = Paths.get(".",
/** * Move a persisted replica from lazypersist directory to a subdirectory * under finalized. */ ReplicaInfo activateSavedReplica(ReplicaInfo replicaInfo, RamDiskReplica replicaState) throws IOException { File metaFile = replicaState.getSavedMetaFile(); File blockFile = replicaState.getSavedBlockFile(); final long blockId = replicaInfo.getBlockId(); final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId); final File targetBlockFile = new File(blockDir, blockFile.getName()); final File targetMetaFile = new File(blockDir, metaFile.getName()); fileIoProvider.moveFile(volume, blockFile, targetBlockFile); FsDatasetImpl.LOG.info("Moved " + blockFile + " to " + targetBlockFile); fileIoProvider.moveFile(volume, metaFile, targetMetaFile); FsDatasetImpl.LOG.info("Moved " + metaFile + " to " + targetMetaFile); ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.FINALIZED) .setBlockId(blockId) .setLength(replicaInfo.getBytesOnDisk()) .setGenerationStamp(replicaInfo.getGenerationStamp()) .setFsVolume(replicaState.getLazyPersistVolume()) .setDirectoryToUse(targetBlockFile.getParentFile()) .build(); return newReplicaInfo; }
.setGenerationStamp(genStamp) .setFsVolume(volume) .setDirectoryToUse(DatanodeUtil.idToBlockDir(finalizedDir, blockId)) .build(); } else {
/** * Return the parent directory path where this replica is located * @return the parent directory path where this replica is located */ File getDir() { return hasSubdirs ? DatanodeUtil.idToBlockDir(baseDir, getBlockId()) : baseDir; }
/** * Return the parent directory path where this replica is located * @return the parent directory path where this replica is located */ File getDir() { return hasSubdirs ? DatanodeUtil.idToBlockDir(baseDir, getBlockId()) : baseDir; }
/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, File srcMeta, File srcFile, File destRoot, boolean calculateChecksum) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); final File dstFile = new File(destDir, srcFile.getName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcMeta, srcFile, dstMeta, dstFile, calculateChecksum); }
File addBlock(Block b, File f) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); if (!blockDir.exists()) { if (!blockDir.mkdirs()) { throw new IOException("Failed to mkdirs " + blockDir); } } File blockFile = FsDatasetImpl.moveBlockFiles(b, f, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length()); return blockFile; }
File addBlock(Block b, File f) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); if (!blockDir.exists()) { if (!blockDir.mkdirs()) { throw new IOException("Failed to mkdirs " + blockDir); } } File blockFile = FsDatasetImpl.moveBlockFiles(b, f, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length()); return blockFile; }
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaUnderRecovery replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return copyBlockFiles(replicaInfo.getMetaFile(), replicaInfo.getBlockFile(), dstMetaFile, dstBlockFile, true); }
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaUnderRecovery replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return copyBlockFiles(replicaInfo.getMetaFile(), replicaInfo.getBlockFile(), dstMetaFile, dstBlockFile, true); }
/** * Get file correpsonding to a block * @param storageDir storage directory * @param blk the block * @return data file corresponding to the block */ public static File getBlockFile(File storageDir, ExtendedBlock blk) { return new File(DatanodeUtil.idToBlockDir(getFinalizedDir(storageDir, blk.getBlockPoolId()), blk.getBlockId()), blk.getBlockName()); }
/** * Get the latest metadata file correpsonding to a block * @param storageDir storage directory * @param blk the block * @return metadata file corresponding to the block */ public static File getBlockMetadataFile(File storageDir, ExtendedBlock blk) { return new File(DatanodeUtil.idToBlockDir(getFinalizedDir(storageDir, blk.getBlockPoolId()), blk.getBlockId()), blk.getBlockName() + "_" + blk.getGenerationStamp() + Block.METADATA_EXTENSION); }