private LocalReplicaInPipeline buildRBW() throws IllegalArgumentException { if (null != fromReplica && fromReplica.getState() == ReplicaState.RBW) { return new ReplicaBeingWritten((ReplicaBeingWritten) fromReplica); } else if (null != fromReplica) { throw new IllegalArgumentException("Incompatible fromReplica " + "state: " + fromReplica.getState()); } else { if (null != block) { if (null == writer) { throw new IllegalArgumentException("A valid writer is " + "required for constructing a RBW from block " + block.getBlockId()); } return new ReplicaBeingWritten(block, volume, directoryUsed, writer); } else { if (length != -1) { return new ReplicaBeingWritten(blockId, length, genStamp, volume, directoryUsed, writer, bytesToReserve); } else { return new ReplicaBeingWritten(blockId, genStamp, volume, directoryUsed, bytesToReserve); } } } }
@Override public long getVisibleLength() { return getBytesAcked(); // all acked bytes are visible }
/** * * @param bytesReserved Space that was reserved during * block creation. Now that the block is being finalized we * can free up this space. * @return * @throws IOException */ ReplicaInfo addFinalizedBlock(String bpid, Block b, ReplicaInfo replicaInfo, long bytesReserved) throws IOException { releaseReservedSpace(bytesReserved); File dest = getBlockPoolSlice(bpid).addFinalizedBlock(b, replicaInfo); byte[] checksum = null; // copy the last partial checksum if the replica is originally // in finalized or rbw state. if (replicaInfo.getState() == ReplicaState.FINALIZED) { FinalizedReplica finalized = (FinalizedReplica)replicaInfo; checksum = finalized.getLastPartialChunkChecksum(); } else if (replicaInfo.getState() == ReplicaState.RBW) { ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; checksum = rbw.getLastChecksumAndDataLen().getChecksum(); } return new ReplicaBuilder(ReplicaState.FINALIZED) .setBlock(replicaInfo) .setFsVolume(this) .setDirectoryToUse(dest.getParentFile()) .setLastPartialChunkChecksum(checksum) .build(); }
rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout()); rbw.setWriter(Thread.currentThread()); long replicaGenerationStamp = rbw.getGenerationStamp(); if (replicaGenerationStamp < b.getGenerationStamp() || replicaGenerationStamp > newGS) { long bytesAcked = rbw.getBytesAcked(); long numBytes = rbw.getNumBytes(); if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd){ throw new ReplicaNotFoundException("Unmatched length replica " + long bytesOnDisk = rbw.getBytesOnDisk(); long blockDataLength = rbw.getBlockFile().length(); if (bytesOnDisk != blockDataLength) { LOG.info("Resetting bytesOnDisk to match blockDataLength (=" + blockDataLength + ") for replica " + rbw); bytesOnDisk = blockDataLength; rbw.setLastChecksumAndDataLen(bytesOnDisk, null); FsVolumeReference ref = rbw.getVolume().obtainReference(); try { final File replicafile = rbw.getBlockFile(); truncateBlock(replicafile, rbw.getMetaFile(), bytesOnDisk, bytesAcked); rbw.setNumBytes(bytesAcked); rbw.setLastChecksumAndDataLen(bytesAcked, null);
long bytesOnDisk = rbw.getBytesOnDisk(); rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null); fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L, rbw.getNumBytes()); Assert.assertTrue(rbw.getBytesOnDisk() == rbw.getBytesAcked()); rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null); try (RandomAccessFile blockRAF = new RandomAccessFile(rbw.getBlockFile(), "rw")) { rbw.getNumBytes()); Assert.fail("recovery should have failed"); } catch (ReplicaNotFoundException rnfe) {
ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout()); rbw.setWriter(Thread.currentThread()); if (replicaLen != rbw.getBytesOnDisk() || replicaLen != rbw.getBytesAcked()) { throw new ReplicaAlreadyExistsException("RBW replica " + replicaInfo + "bytesRcvd(" + rbw.getNumBytes() + "), bytesOnDisk(" + rbw.getBytesOnDisk() + "), and bytesAcked(" + rbw.getBytesAcked() + ") are not the same.");
bpslice.getRbwDir()); final ReplicaBeingWritten rbw = new ReplicaBeingWritten( blockId, numBytes, expectedGs, v, dest.getParentFile(), Thread.currentThread(), 0); rbw.setBytesAcked(visible); b.getGenerationStamp()); byte[] lastChunkChecksum = v.loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum);
ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten( replicaInfo.getBlockId(), replicaInfo.getNumBytes(), newGS, v, newBlkFile.getParentFile(), Thread.currentThread(), bytesReserved); newReplicaInfo.setLastChecksumAndDataLen( replicaInfo.getNumBytes(), lastChunkChecksum); File newmeta = newReplicaInfo.getMetaFile();
final ExtendedBlock b = new ExtendedBlock(bpid, oldrbw.getBlockId(), oldrbw.getBytesAcked(), oldrbw.getGenerationStamp()); final BlockOpResponseProto s = DFSTestUtil.transferRbw( b, DFSClientAdapter.getDFSClient(fs), oldnodeinfo, newnodeinfo); Assert.assertEquals(oldrbw.getBlockId(), newrbw.getBlockId()); Assert.assertEquals(oldrbw.getGenerationStamp(), newrbw.getGenerationStamp()); Assert.assertEquals(oldrbw.getVisibleLength(), newrbw.getVisibleLength());
"New block file: " + blockPath + " must be on " + "same volume as recovery replica: " + volumePath; ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten( newBlockId, recoveryId, volume, blockFile.getParentFile(), newlength); newReplicaInfo.setNumBytes(newlength);
(ReplicaBeingWritten) replicaHandler.getReplica(); ReplicaOutputStreams outputStreams = rbw.createStreams(false, DEFAULT_CHECKSUM); OutputStream dataOutput = outputStreams.getDataOut(); FsDatasetUtil.computeChecksum(rbw.getMetaFile(), rbw.getMetaFile(), rbw.getBlockFile());
/** * Wait for rbw replica to reach the length * @param rbw replica that is being written to * @param len minimum length to reach * @throws IOException on failing to reach the len in given wait time */ private static void waitForMinLength(ReplicaBeingWritten rbw, long len) throws IOException { // Wait for 3 seconds for rbw replica to reach the minimum length for (int i = 0; i < 30 && rbw.getBytesOnDisk() < len; i++) { try { Thread.sleep(100); } catch (InterruptedException ie) { throw new IOException(ie); } } long bytesOnDisk = rbw.getBytesOnDisk(); if (bytesOnDisk < len) { throw new IOException( String.format("Need %d bytes, but only %d bytes available", len, bytesOnDisk)); } }
replicaBeingWritten.setBytesAcked(pkt.offsetInBlock);
rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout()); rbw.setWriter(Thread.currentThread()); long replicaGenerationStamp = rbw.getGenerationStamp(); if (replicaGenerationStamp < b.getGenerationStamp() || replicaGenerationStamp > newGS) { long bytesAcked = rbw.getBytesAcked(); long numBytes = rbw.getNumBytes(); if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd){ throw new ReplicaNotFoundException("Unmatched length replica " + long bytesOnDisk = rbw.getBytesOnDisk(); long blockDataLength = rbw.getBlockFile().length(); if (bytesOnDisk != blockDataLength) { LOG.info("Resetting bytesOnDisk to match blockDataLength (=" + blockDataLength + ") for replica " + rbw); bytesOnDisk = blockDataLength; rbw.setLastChecksumAndDataLen(bytesOnDisk, null); FsVolumeReference ref = rbw.getVolume().obtainReference(); try { final File replicafile = rbw.getBlockFile(); truncateBlock(replicafile, rbw.getMetaFile(), bytesOnDisk, bytesAcked); rbw.setNumBytes(bytesAcked); rbw.setLastChecksumAndDataLen(bytesAcked, null);
ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout()); rbw.setWriter(Thread.currentThread()); if (replicaLen != rbw.getBytesOnDisk() || replicaLen != rbw.getBytesAcked()) { throw new ReplicaAlreadyExistsException("RBW replica " + replicaInfo + "bytesRcvd(" + rbw.getNumBytes() + "), bytesOnDisk(" + rbw.getBytesOnDisk() + "), and bytesAcked(" + rbw.getBytesAcked() + ") are not the same.");
bpslice.getRbwDir()); final ReplicaBeingWritten rbw = new ReplicaBeingWritten( blockId, numBytes, expectedGs, v, dest.getParentFile(), Thread.currentThread(), 0); rbw.setBytesAcked(visible); b.getGenerationStamp()); byte[] lastChunkChecksum = v.loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum);
ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten( replicaInfo.getBlockId(), replicaInfo.getNumBytes(), newGS, v, newBlkFile.getParentFile(), Thread.currentThread(), bytesReserved); newReplicaInfo.setLastChecksumAndDataLen( replicaInfo.getNumBytes(), lastChunkChecksum); File newmeta = newReplicaInfo.getMetaFile();
"New block file: " + blockPath + " must be on " + "same volume as recovery replica: " + volumePath; ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten( newBlockId, recoveryId, volume, blockFile.getParentFile(), newlength); newReplicaInfo.setNumBytes(newlength);
/** * Wait for rbw replica to reach the length * @param rbw replica that is being written to * @param len minimum length to reach * @throws IOException on failing to reach the len in given wait time */ private static void waitForMinLength(ReplicaBeingWritten rbw, long len) throws IOException { // Wait for 3 seconds for rbw replica to reach the minimum length for (int i = 0; i < 30 && rbw.getBytesOnDisk() < len; i++) { try { Thread.sleep(100); } catch (InterruptedException ie) { throw new IOException(ie); } } long bytesOnDisk = rbw.getBytesOnDisk(); if (bytesOnDisk < len) { throw new IOException( String.format("Need %d bytes, but only %d bytes available", len, bytesOnDisk)); } }
newReplica = new ReplicaBeingWritten(blockId, validateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.getParentFile(), null, 0);