private void deleteReplica(final ReplicaInfo replicaToDelete) { // Delete the files on disk. Failure here is okay. final File blockFile = replicaToDelete.getBlockFile(); if (!blockFile.delete()) { LOG.warn("Failed to delete block file " + blockFile); } final File metaFile = replicaToDelete.getMetaFile(); if (!metaFile.delete()) { LOG.warn("Failed to delete meta file " + metaFile); } }
/** Delete a block file */ private long deleteBlockFile() { synchronized(fds) { for (ReplicaInfo b : fds.volumeMap.replicas()) { File f = b.getBlockFile(); File mf = b.getMetaFile(); // Delete a block file that has corresponding metadata file if (f.exists() && mf.exists() && f.delete()) { LOG.info("Deleting block file " + f.getAbsolutePath()); return b.getBlockId(); } } } return 0; }
/** Truncate a block file */ private long truncateBlockFile() throws IOException { synchronized (fds) { for (ReplicaInfo b : fds.volumeMap.replicas()) { File f = b.getBlockFile(); File mf = b.getMetaFile(); // Truncate a block file that has a corresponding metadata file if (f.exists() && f.length() != 0 && mf.exists()) { FileOutputStream s = new FileOutputStream(f); FileChannel channel = s.getChannel(); channel.truncate(0); LOG.info("Truncated block file " + f.getAbsolutePath()); return b.getBlockId(); } } } return 0; }
/** Check the files of a replica. */ static void checkReplicaFiles(final ReplicaInfo r) throws IOException { //check replica's file final File f = r.getBlockFile(); if (!f.exists()) { throw new FileNotFoundException("File " + f + " not found, r=" + r); } if (r.getBytesOnDisk() != f.length()) { throw new IOException("File length mismatched. The length of " + f + " is " + f.length() + " but r=" + r); } //check replica's meta file final File metafile = FsDatasetUtil.getMetaFile(f, r.getGenerationStamp()); if (!metafile.exists()) { throw new IOException("Metafile " + metafile + " does not exist, r=" + r); } if (metafile.length() == 0) { throw new IOException("Metafile " + metafile + " is empty, r=" + r); } }
/** Check the files of a replica. */ static void checkReplicaFiles(final ReplicaInfo r) throws IOException { //check replica's file final File f = r.getBlockFile(); if (!f.exists()) { throw new FileNotFoundException("File " + f + " not found, r=" + r); } if (r.getBytesOnDisk() != f.length()) { throw new IOException("File length mismatched. The length of " + f + " is " + f.length() + " but r=" + r); } //check replica's meta file final File metafile = FsDatasetUtil.getMetaFile(f, r.getGenerationStamp()); if (!metafile.exists()) { throw new IOException("Metafile " + metafile + " does not exist, r=" + r); } if (metafile.length() == 0) { throw new IOException("Metafile " + metafile + " is empty, r=" + r); } }
private void deleteReplica(final ReplicaInfo replicaToDelete) { // Delete the files on disk. Failure here is okay. final File blockFile = replicaToDelete.getBlockFile(); if (!blockFile.delete()) { LOG.warn("Failed to delete block file " + blockFile); } final File metaFile = replicaToDelete.getMetaFile(); if (!metaFile.delete()) { LOG.warn("Failed to delete meta file " + metaFile); } }
@Override //Object public String toString() { return getClass().getSimpleName() + ", " + super.toString() + ", " + getState() + "\n getNumBytes() = " + getNumBytes() + "\n getBytesOnDisk() = " + getBytesOnDisk() + "\n getVisibleLength()= " + getVisibleLength() + "\n getVolume() = " + getVolume() + "\n getBlockFile() = " + getBlockFile(); }
@Override //Object public String toString() { return getClass().getSimpleName() + ", " + super.toString() + ", " + getState() + "\n getNumBytes() = " + getNumBytes() + "\n getBytesOnDisk() = " + getBytesOnDisk() + "\n getVisibleLength()= " + getVisibleLength() + "\n getVolume() = " + getVolume() + "\n getBlockFile() = " + getBlockFile(); }
/** Delete a block file */ private long deleteBlockFile() { synchronized(fds) { for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) { File f = b.getBlockFile(); File mf = b.getMetaFile(); // Delete a block file that has corresponding metadata file if (f.exists() && mf.exists() && f.delete()) { LOG.info("Deleting block file " + f.getAbsolutePath()); return b.getBlockId(); } } } return 0; }
/** Truncate a block file */ private long truncateBlockFile() throws IOException { synchronized (fds) { for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) { File f = b.getBlockFile(); File mf = b.getMetaFile(); // Truncate a block file that has a corresponding metadata file if (f.exists() && f.length() != 0 && mf.exists()) { FileOutputStream s = null; FileChannel channel = null; try { s = new FileOutputStream(f); channel = s.getChannel(); channel.truncate(0); LOG.info("Truncated block file " + f.getAbsolutePath()); return b.getBlockId(); } finally { IOUtils.cleanup(LOG, channel, s); } } } } return 0; }
private static void createUnlinkTmpFile(ReplicaInfo replicaInfo, boolean changeBlockFile, boolean isRename) throws IOException { File src; if (changeBlockFile) { src = replicaInfo.getBlockFile(); } else { src = replicaInfo.getMetaFile(); } File dst = DatanodeUtil.getUnlinkTmpFile(src); if (isRename) { src.renameTo(dst); } else { FileInputStream in = new FileInputStream(src); try { FileOutputStream out = new FileOutputStream(dst); try { IOUtils.copyBytes(in, out, 1); } finally { out.close(); } } finally { in.close(); } } }
private void setup(FSDataset dataSet) throws IOException { // setup replicas map ReplicasMap replicasMap = dataSet.volumeMap; FSVolume vol = dataSet.volumes.getNextVolume(0); ReplicaInfo replicaInfo = new FinalizedReplica( blocks[FINALIZED], vol, vol.getDir()); replicasMap.add(replicaInfo); replicaInfo.getBlockFile().createNewFile(); replicaInfo.getMetaFile().createNewFile(); replicasMap.add(new ReplicaInPipeline( blocks[TEMPORARY].getBlockId(), blocks[TEMPORARY].getGenerationStamp(), vol, vol.createTmpFile(blocks[TEMPORARY]).getParentFile())); replicaInfo = new ReplicaBeingWritten(blocks[RBW], vol, vol.createRbwFile(blocks[RBW]).getParentFile(), null); replicasMap.add(replicaInfo); replicaInfo.getBlockFile().createNewFile(); replicaInfo.getMetaFile().createNewFile(); replicasMap.add(new ReplicaWaitingToBeRecovered(blocks[RWR], vol, vol.createRbwFile(blocks[RWR]).getParentFile())); replicasMap.add(new ReplicaUnderRecovery( new FinalizedReplica(blocks[RUR], vol, vol.getDir()), 2007)); }
/** * Turn the block identifier into a filename * @param bpid Block pool Id * @param blockId a block's id * @return on disk data file path; null if the replica does not exist */ File getFile(final String bpid, final long blockId, boolean touch) { ReplicaInfo info = volumeMap.get(bpid, blockId); if (info != null) { if (touch && info.getVolume().isTransientStorage()) { ramDiskReplicaTracker.touch(bpid, blockId); datanode.getMetrics().incrRamDiskBlocksReadHits(); } return info.getBlockFile(); } return null; }
/** * Turn the block identifier into a filename * @param bpid Block pool Id * @param blockId a block's id * @return on disk data file path; null if the replica does not exist */ File getFile(final String bpid, final long blockId, boolean touch) { ReplicaInfo info = volumeMap.get(bpid, blockId); if (info != null) { if (touch && info.getVolume().isTransientStorage()) { ramDiskReplicaTracker.touch(bpid, blockId); datanode.getMetrics().incrRamDiskBlocksReadHits(); } return info.getBlockFile(); } return null; }
@VisibleForTesting static ReplicaInfo selectReplicaToDelete(final ReplicaInfo replica1, final ReplicaInfo replica2) { ReplicaInfo replicaToKeep; ReplicaInfo replicaToDelete; // it's the same block so don't ever delete it, even if GS or size // differs. caller should keep the one it just discovered on disk if (replica1.getBlockFile().equals(replica2.getBlockFile())) { return null; } if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) { replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp() ? replica1 : replica2; } else if (replica1.getNumBytes() != replica2.getNumBytes()) { replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ? replica1 : replica2; } else if (replica1.getVolume().isTransientStorage() && !replica2.getVolume().isTransientStorage()) { replicaToKeep = replica2; } else { replicaToKeep = replica1; } replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1; if (LOG.isDebugEnabled()) { LOG.debug("resolveDuplicateReplicas decide to keep " + replicaToKeep + ". Will try to delete " + replicaToDelete); } return replicaToDelete; }
private synchronized FinalizedReplica finalizeReplica(String bpid, ReplicaInfo replicaInfo) throws IOException { FinalizedReplica newReplicaInfo = null; if (replicaInfo.getState() == ReplicaState.RUR && ((ReplicaUnderRecovery)replicaInfo).getOriginalReplica().getState() == ReplicaState.FINALIZED) { newReplicaInfo = (FinalizedReplica) ((ReplicaUnderRecovery)replicaInfo).getOriginalReplica(); } else { FsVolumeImpl v = (FsVolumeImpl)replicaInfo.getVolume(); File f = replicaInfo.getBlockFile(); if (v == null) { throw new IOException("No volume for temporary file " + f + " for block " + replicaInfo); } File dest = v.addFinalizedBlock( bpid, replicaInfo, f, replicaInfo.getBytesReserved()); newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile()); if (v.isTransientStorage()) { ramDiskReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v); datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes()); } } volumeMap.add(bpid, newReplicaInfo); return newReplicaInfo; }
/** * Returns handles to the block file and its metadata file */ @Override // FsDatasetSpi public synchronized ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkOffset, long metaOffset) throws IOException { ReplicaInfo info = getReplicaInfo(b); FsVolumeReference ref = info.getVolume().obtainReference(); try { InputStream blockInStream = openAndSeek(info.getBlockFile(), blkOffset); try { InputStream metaInStream = openAndSeek(info.getMetaFile(), metaOffset); return new ReplicaInputStreams(blockInStream, metaInStream, ref); } catch (IOException e) { IOUtils.cleanup(null, blockInStream); throw e; } } catch (IOException e) { IOUtils.cleanup(null, ref); throw e; } }
/** * Returns handles to the block file and its metadata file */ @Override // FsDatasetSpi public synchronized ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkOffset, long metaOffset) throws IOException { ReplicaInfo info = getReplicaInfo(b); FsVolumeReference ref = info.getVolume().obtainReference(); try { InputStream blockInStream = openAndSeek(info.getBlockFile(), blkOffset); try { InputStream metaInStream = openAndSeek(info.getMetaFile(), metaOffset); return new ReplicaInputStreams(blockInStream, metaInStream, ref); } catch (IOException e) { IOUtils.cleanup(null, blockInStream); throw e; } } catch (IOException e) { IOUtils.cleanup(null, ref); throw e; } }
/** * Remove the temporary block file (if any) */ @Override // FsDatasetSpi public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo.getBlockFile(), replicaInfo.getMetaFile(), b.getLocalBlock())) { LOG.warn("Block " + b + " unfinalized and removed. " ); } if (replicaInfo.getVolume().isTransientStorage()) { ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true); } } }
/** * Remove the temporary block file (if any) */ @Override // FsDatasetSpi public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo.getBlockFile(), replicaInfo.getMetaFile(), b.getLocalBlock())) { LOG.warn("Block " + b + " unfinalized and removed. " ); } if (replicaInfo.getVolume().isTransientStorage()) { ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true); } } }