private LocalReplica buildFinalizedReplica() throws IllegalArgumentException { if (null != fromReplica && fromReplica.getState() == ReplicaState.FINALIZED) { return new FinalizedReplica((FinalizedReplica)fromReplica); } else if (null != this.fromReplica) { throw new IllegalArgumentException("Incompatible fromReplica " + "state: " + fromReplica.getState()); } else { if (null != block) { return new FinalizedReplica(block, volume, directoryUsed, lastPartialChunkChecksum); } else { return new FinalizedReplica(blockId, length, genStamp, volume, directoryUsed, lastPartialChunkChecksum); } } }
public void loadLastPartialChunkChecksum() throws IOException { byte[] lastChecksum = getVolume().loadLastPartialChunkChecksum( getBlockFile(), getMetaFile()); setLastPartialChunkChecksum(lastChecksum); } }
@Override public long getBytesOnDisk() { return getNumBytes(); }
private ChunkChecksum getPartialChunkChecksumForFinalized( FinalizedReplica finalized) throws IOException { // There are a number of places in the code base where a finalized replica // object is created. If last partial checksum is loaded whenever a // finalized replica is created, it would increase latency in DataNode // initialization. Therefore, the last partial chunk checksum is loaded // lazily. // Load last checksum in case the replica is being written concurrently final long replicaVisibleLength = replica.getVisibleLength(); if (replicaVisibleLength % CHUNK_SIZE != 0 && finalized.getLastPartialChunkChecksum() == null) { // the finalized replica does not have precomputed last partial // chunk checksum. Recompute now. try { finalized.loadLastPartialChunkChecksum(); return new ChunkChecksum(finalized.getVisibleLength(), finalized.getLastPartialChunkChecksum()); } catch (FileNotFoundException e) { // meta file is lost. Continue anyway to preserve existing behavior. DataNode.LOG.warn( "meta file " + finalized.getMetaFile() + " is missing!"); return null; } } else { // If the checksum is null, BlockSender will use on-disk checksum. return new ChunkChecksum(finalized.getVisibleLength(), finalized.getLastPartialChunkChecksum()); } }
finalized.getVisibleLength(), finalized.getLastPartialChunkChecksum());
throws IOException { cacheManager.uncacheBlock(bpid, replicaInfo.getBlockId()); replicaInfo.unlinkBlock(1); File blkfile = replicaInfo.getBlockFile(); FsVolumeImpl v = (FsVolumeImpl)replicaInfo.getVolume(); long bytesReserved = estimateBlockLen - replicaInfo.getNumBytes(); if (v.getAvailable() < bytesReserved) { throw new DiskOutOfSpaceException("Insufficient space for appending to " + replicaInfo); File newBlkFile = new File(v.getRbwDir(bpid), replicaInfo.getBlockName()); File oldmeta = replicaInfo.getMetaFile(); ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten( replicaInfo.getBlockId(), replicaInfo.getNumBytes(), newGS, v, newBlkFile.getParentFile(), Thread.currentThread(), bytesReserved); replicaInfo.getBlockFile(), replicaInfo.getMetaFile()); newReplicaInfo.setLastChecksumAndDataLen( replicaInfo.getNumBytes(), lastChunkChecksum);
FinalizedReplica memBlock = memReport[m]; ScanInfo info = blockpoolReport[d]; if (info.getBlockId() < memBlock.getBlockId()) { if (!dataset.isDeletingBlock(bpid, info.getBlockId())) { continue; if (info.getBlockId() > memBlock.getBlockId()) { memBlock.getBlockId(), info.getVolume()); m++; continue; } else if (info.getGenStamp() != memBlock.getGenerationStamp() || info.getBlockFileLength() != memBlock.getNumBytes()) { } else if (info.getBlockFile().compareTo(memBlock.getBlockFile()) != 0) { FinalizedReplica current = memReport[m++]; addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume());
/** * gets the last chunk checksum and the length of the block corresponding * to that checksum. * Note, need to be called with the FsDataset lock acquired. May improve to * lock only the FsVolume in the future. * @throws IOException */ public ChunkChecksum getLastChecksumAndDataLen() throws IOException { ChunkChecksum chunkChecksum = null; try { byte[] lastChecksum = getVolume().loadLastPartialChunkChecksum( getBlockFile(), getMetaFile()); if (lastChecksum != null) { chunkChecksum = new ChunkChecksum(getVisibleLength(), lastChecksum); } } catch (FileNotFoundException e) { // meta file is lost. Try to continue anyway. DataNode.LOG.warn("meta file " + getMetaFile() + " is missing!"); } catch (IOException ioe) { DataNode.LOG.warn("Unable to read checksum from meta file " + getMetaFile(), ioe); } return chunkChecksum; } }
assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == recoveryId && finalized.getNumBytes() == newlength : "Replica information mismatched: oldBlock=" + oldBlock + ", recoveryId=" + recoveryId + ", newlength=" + newlength + ", newBlockId=" + newBlockId + ", finalized=" + finalized; } else { assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == oldBlock.getGenerationStamp() && finalized.getNumBytes() == oldBlock.getNumBytes() : "Finalized and old information mismatched: oldBlock=" + oldBlock + ", genStamp=" + oldBlock.getGenerationStamp()
chunkChecksum = frep.getLastChecksumAndDataLen();
throws IOException { cacheManager.uncacheBlock(bpid, replicaInfo.getBlockId()); replicaInfo.unlinkBlock(1); File blkfile = replicaInfo.getBlockFile(); FsVolumeImpl v = (FsVolumeImpl)replicaInfo.getVolume(); long bytesReserved = estimateBlockLen - replicaInfo.getNumBytes(); if (v.getAvailable() < bytesReserved) { throw new DiskOutOfSpaceException("Insufficient space for appending to " + replicaInfo); File newBlkFile = new File(v.getRbwDir(bpid), replicaInfo.getBlockName()); File oldmeta = replicaInfo.getMetaFile(); ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten( replicaInfo.getBlockId(), replicaInfo.getNumBytes(), newGS, v, newBlkFile.getParentFile(), Thread.currentThread(), bytesReserved); replicaInfo.getBlockFile(), replicaInfo.getMetaFile()); newReplicaInfo.setLastChecksumAndDataLen( replicaInfo.getNumBytes(), lastChunkChecksum);
FinalizedReplica memBlock = memReport[m]; ScanInfo info = blockpoolReport[d]; if (info.getBlockId() < memBlock.getBlockId()) { if (!dataset.isDeletingBlock(bpid, info.getBlockId())) { continue; if (info.getBlockId() > memBlock.getBlockId()) { memBlock.getBlockId(), info.getVolume()); m++; continue; } else if (info.getGenStamp() != memBlock.getGenerationStamp() || info.getBlockFileLength() != memBlock.getNumBytes()) { } else if (info.getBlockFile().compareTo(memBlock.getBlockFile()) != 0) { FinalizedReplica current = memReport[m++]; addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume());
/** * gets the last chunk checksum and the length of the block corresponding * to that checksum. * Note, need to be called with the FsDataset lock acquired. May improve to * lock only the FsVolume in the future. * @throws IOException */ public ChunkChecksum getLastChecksumAndDataLen() throws IOException { ChunkChecksum chunkChecksum = null; try { byte[] lastChecksum = getVolume().loadLastPartialChunkChecksum( getBlockFile(), getMetaFile()); if (lastChecksum != null) { chunkChecksum = new ChunkChecksum(getVisibleLength(), lastChecksum); } } catch (FileNotFoundException e) { // meta file is lost. Try to continue anyway. DataNode.LOG.warn("meta file " + getMetaFile() + " is missing!"); } catch (IOException ioe) { DataNode.LOG.warn("Unable to read checksum from meta file " + getMetaFile(), ioe); } return chunkChecksum; } }
assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == recoveryId && finalized.getNumBytes() == newlength : "Replica information mismatched: oldBlock=" + oldBlock + ", recoveryId=" + recoveryId + ", newlength=" + newlength + ", newBlockId=" + newBlockId + ", finalized=" + finalized; } else { assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == oldBlock.getGenerationStamp() && finalized.getNumBytes() == oldBlock.getNumBytes() : "Finalized and old information mismatched: oldBlock=" + oldBlock + ", genStamp=" + oldBlock.getGenerationStamp()
chunkChecksum = frep.getLastChecksumAndDataLen();
private static ReplicaInfo createReplicaInfo(Block b) { return new FinalizedReplica(b, null, null); }
@Override public long getVisibleLength() { return getNumBytes(); // all bytes are visible }
@Override // FSDatasetInterface public FinalizedReplica updateReplicaUnderRecovery(Block oldBlock, long recoveryId, long newlength) throws IOException { return new FinalizedReplica( oldBlock.getBlockId(), newlength, recoveryId, null, null); }
@Override public long getVisibleLength() { return getNumBytes(); // all bytes are visible }
private static ReplicaInfo createReplicaInfo(Block b) { return new FinalizedReplica(b, null, null); }