long getBlockLen(int i) { return StripedBlockUtil.getInternalBlockLength(blockGroup.getNumBytes(), ecPolicy, i); }
OpWriteBlockProto.Builder writeBlockProtoBuilder = OpWriteBlockProto.newBuilder() .setHeader(header).setStage(OpWriteBlockProto.BlockConstructionStage.valueOf(stage.name())) .setPipelineSize(1).setMinBytesRcvd(locatedBlock.getBlock().getNumBytes()) .setMaxBytesRcvd(maxBytesRcvd).setLatestGenerationStamp(latestGS) .setRequestedChecksum(checksumProto)
/** Convert an ExtendedBlock to a Json map. */ private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) { if (extendedblock == null) { return null; } final Map<String, Object> m = new TreeMap<String, Object>(); m.put("blockPoolId", extendedblock.getBlockPoolId()); m.put("blockId", extendedblock.getBlockId()); m.put("numBytes", extendedblock.getNumBytes()); m.put("generationStamp", extendedblock.getGenerationStamp()); return m; }
BlockChecksumComputer(DataNode datanode, ExtendedBlock block, BlockChecksumOptions blockChecksumOptions) throws IOException { super(datanode, blockChecksumOptions); this.block = block; this.requestLength = block.getNumBytes(); Preconditions.checkArgument(requestLength >= 0); this.metadataIn = datanode.data.getMetaDataInputStream(block); this.visibleLength = datanode.data.getReplicaVisibleLength(block); this.partialBlk = requestLength < visibleLength; int ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(datanode.getConf()); this.checksumIn = new DataInputStream( new BufferedInputStream(metadataIn, ioFileBufferSize)); }
/** * Connect to the first item in the target list. Pass along the * entire target list, the block, and the data. */ DataTransfer(DatanodeInfo targets[], StorageType[] targetStorageTypes, String[] targetStorageIds, ExtendedBlock b, BlockConstructionStage stage, final String clientname) { if (DataTransferProtocol.LOG.isDebugEnabled()) { DataTransferProtocol.LOG.debug("{}: {} (numBytes={}), stage={}, " + "clientname={}, targets={}, target storage types={}, " + "target storage IDs={}", getClass().getSimpleName(), b, b.getNumBytes(), stage, clientname, Arrays.asList(targets), targetStorageTypes == null ? "[]" : Arrays.asList(targetStorageTypes), targetStorageIds == null ? "[]" : Arrays.asList(targetStorageIds)); } this.targets = targets; this.targetStorageTypes = targetStorageTypes; this.targetStorageIds = targetStorageIds; this.b = b; this.stage = stage; BPOfferService bpos = blockPoolManager.get(b.getBlockPoolId()); bpReg = bpos.bpRegistration; this.clientname = clientname; this.cachingStrategy = new CachingStrategy(true, getDnConf().readaheadLength); }
public void updateLastBlock(BlockInfo lastBlock, ExtendedBlock newBlock) { lastBlock.setNumBytes(newBlock.getNumBytes()); List<ReplicaUnderConstruction> staleReplicas = lastBlock .setGenerationStampAndVerifyReplicas(newBlock.getGenerationStamp()); removeStaleReplicas(staleReplicas, lastBlock); }
private ExtendedBlock getInternalBlock(int numDataUnits, int idx) { // Sets requested number of bytes in blockGroup which is required to // construct the internal block for computing checksum. long actualNumBytes = blockGroup.getNumBytes(); blockGroup.setNumBytes(requestedNumBytes); ExtendedBlock block = StripedBlockUtil.constructInternalBlock(blockGroup, ecPolicy.getCellSize(), numDataUnits, idx); // Set back actualNumBytes value in blockGroup. blockGroup.setNumBytes(actualNumBytes); return block; }
private BlockReader createBlockReader(long offsetInBlock) { if (offsetInBlock >= block.getNumBytes()) { return null; block.getNumBytes() - offsetInBlock, true, "", peer, source, null, stripedReader.getCachingStrategy(), -1); } catch (IOException e) {
.getBlockPoolId()); ClientTraceLog.info(String.format(DN_CLIENTTRACE_FORMAT, inAddr, myAddr, block.getNumBytes(), "HDFS_WRITE", clientname, offset, dnR.getDatanodeUuid(), block, endTime - startTime)); } else { LOG.info("Received " + block + " size " + block.getNumBytes() + " from " + inAddr);
throw new IOException(msg); if (newBlock.getNumBytes() < lastBlock.getNumBytes()) { final String msg = "Update " + oldBlock + " (size=" + oldBlock.getNumBytes() + ") to a smaller size block " + newBlock + " (size=" + newBlock.getNumBytes() + ")"; LOG.warn(msg); throw new IOException(msg);
parityBlkNum = stripedReconInfo.getEcPolicy().getNumParityUnits(); int cellsNum = (int) ((stripedReconInfo.getBlockGroup().getNumBytes() - 1) / stripedReconInfo.getEcPolicy().getCellSize() + 1); minRequiredSources = Math.min(cellsNum, dataBlkNum);
ReplicaNotFoundException.UNFINALIZED_REPLICA + block); if (replicaInfo.getNumBytes() != block.getNumBytes()) { throw new IOException("Corrupted replica " + replicaInfo + " with a length of " + replicaInfo.getNumBytes() + " expected length is " + block.getNumBytes()); try (AutoCloseableLock lock = datasetLock.acquire()) { volumeRef = volumes.getNextVolume(targetStorageType, targetStorageId, block.getNumBytes());
public ReplicaInPipeline createRbw(ExtendedBlock b) throws IOException { File f = createRbwFile(b.getBlockPoolId(), b.getLocalBlock()); LocalReplicaInPipeline newReplicaInfo = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(b.getBlockId()) .setGenerationStamp(b.getGenerationStamp()) .setFsVolume(this) .setDirectoryToUse(f.getParentFile()) .setBytesToReserve(b.getNumBytes()) .buildLocalReplicaInPipeline(); return newReplicaInfo; }
try { replica = append(b.getBlockPoolId(), replicaInfo, newGS, b.getNumBytes()); } catch (IOException e) { IOUtils.cleanup(null, ref);
if (replica.getBytesOnDisk() != oldBlock.getNumBytes()) { throw new IOException("THIS IS NOT SUPPOSED TO HAPPEN:" + " replica.getBytesOnDisk() != block.getNumBytes(), block=" assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == oldBlock.getGenerationStamp() && finalized.getNumBytes() == oldBlock.getNumBytes() : "Finalized and old information mismatched: oldBlock=" + oldBlock + ", genStamp=" + oldBlock.getGenerationStamp() + ", len=" + oldBlock.getNumBytes() + ", finalized=" + finalized;
LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + ", newGS=" + newBlock.getGenerationStamp() + ", newLength=" + newBlock.getNumBytes() + ", newNodes=" + Arrays.asList(newNodes) + ", client=" + clientName
if (isLessThanNeeded(block.getNumBytes(), item)) { return block;
extBlock.getNumBytes(), ecPolicy, blkIndex); Block blk = new Block(ExtendedBlock.getLocalBlock(extBlock)); long blkId = blk.getBlockId() + blkIndex;
newGS, b.getNumBytes()); } else { //RBW replicaInfo.bumpReplicaGS(newGS);
public ReplicaInPipeline convertTemporaryToRbw(ExtendedBlock b, ReplicaInfo temp) throws IOException { final long blockId = b.getBlockId(); final long expectedGs = b.getGenerationStamp(); final long visible = b.getNumBytes(); final long numBytes = temp.getNumBytes(); // move block files to the rbw directory BlockPoolSlice bpslice = getBlockPoolSlice(b.getBlockPoolId()); final File dest = FsDatasetImpl.moveBlockFiles(b.getLocalBlock(), temp, bpslice.getRbwDir()); // create RBW final LocalReplicaInPipeline rbw = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(blockId) .setLength(numBytes) .setGenerationStamp(expectedGs) .setFsVolume(this) .setDirectoryToUse(dest.getParentFile()) .setWriterThread(Thread.currentThread()) .setBytesToReserve(0) .buildLocalReplicaInPipeline(); rbw.setBytesAcked(visible); // load last checksum and datalen final File destMeta = FsDatasetUtil.getMetaFile(dest, b.getGenerationStamp()); byte[] lastChunkChecksum = loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum); return rbw; }