BlockSender blockSender = null; try { blockSender = new BlockSender(block, 0, -1, false, true, true, datanode, null, CachingStrategy.newDropBehind()); throttler.setBandwidth(bytesPerSec); long bytesRead = blockSender.sendBlock(nullStream, null, throttler); resultHandler.handle(block, null); metrics.incrBlocksVerified();
if (isLongRead() && ris.getDataInFd() != null) { manageOsCache(); blockInPosition = fileChannel.position(); streamForSendChunks = baseStream; maxChunksPerPacket = numberOfChunks(TRANSFERTO_BUFFER_SIZE); } else { maxChunksPerPacket = Math.max(1, numberOfChunks(IO_FILE_BUFFER_SIZE)); manageOsCache(); long len = sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler); offset += len; totalRead += len + (numberOfChunks(len) * checksumSize); seqno++; try { sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler); out.flush(); } catch (IOException e) { //socket error throw ioeToSocketException(e); close();
blockSender = new BlockSender(block, blockOffset, length, true, false, sendChecksum, datanode, clientTraceFmt, cachingStrategy); read = blockSender.sendBlock(out, baseStream, null); // send data long duration = Time.monotonicNow() - beginRead; if (blockSender.didSendEntireByteRange()) {
private void writeSuccessWithChecksumInfo(BlockSender blockSender, DataOutputStream out) throws IOException { ReadOpChecksumInfoProto ckInfo = ReadOpChecksumInfoProto.newBuilder() .setChecksum(DataTransferProtoUtil.toProto(blockSender.getChecksum())) .setChunkOffset(blockSender.getOffset()) .build(); BlockOpResponseProto response = BlockOpResponseProto.newBuilder() .setStatus(SUCCESS) .setReadOpChecksumInfo(ckInfo) .build(); response.writeDelimitedTo(out); out.flush(); }
(chunkSize * (long) maxChunks)); int numChunks = numberOfChunks(dataLen); // Number of chunks be sent in the packet int checksumDataLen = numChunks * checksumSize; int packetLen = dataLen + checksumDataLen + 4; int headerLen = writePacketHeader(pkt, dataLen, packetLen); readChecksum(buf, checksumOff, checksumDataLen); verifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff); throw ioeToSocketException(e);
DFSUtilClient.getSmallBufferSize(getConf()))); in = new DataInputStream(unbufIn); blockSender = new BlockSender(b, 0, b.getNumBytes(), false, false, true, DataNode.this, null, cachingStrategy); DatanodeInfo srcNode = new DatanodeInfoBuilder().setNodeID(bpReg) new Sender(out).writeBlock(b, targetStorageTypes[0], accessToken, clientname, targets, targetStorageTypes, srcNode, stage, 0, 0, 0, 0, blockSender.getChecksum(), cachingStrategy, false, false, null, storageId, targetStorageIds); blockSender.sendBlock(out, unbufOut, null);
try { try { blockSender = new BlockSender(block, startOffset, length, true, true, false, datanode, clientTraceFmt); } catch(IOException e) { long read = blockSender.sendBlock(out, baseStream, null); // send data if (blockSender.isBlockReadFully()) {
throw ioeToSocketException(e); long len = sendChunks(pktBuf, maxChunksPerPacket, streamForSendChunks); offset += len; out.flush(); } catch (IOException e) { //socket error throw ioeToSocketException(e); ClientTraceLog.info(String.format(clientTraceFmt, totalRead)); close();
final long replicaVisibleLength; try(AutoCloseableLock lock = datanode.data.acquireDatasetLock()) { replica = getReplica(block, datanode); replicaVisibleLength = replica.getVisibleLength(); waitForMinLength(rbw, startOffset + length); chunkChecksum = rbw.getLastChecksumAndDataLen(); chunkChecksum = getPartialChunkChecksumForFinalized( (FinalizedReplica)replica);
final long replicaVisibleLength; synchronized(datanode.data) { replica = getReplica(block, datanode); replicaVisibleLength = replica.getVisibleLength(); if (replica instanceof FinalizedReplica) { waitForMinLength(rbw, startOffset + length); chunkChecksum = rbw.getLastChecksumAndDataLen();
/** * sendBlock() is used to read block and its metadata and stream the data to * either a client or to another datanode. * * @param out stream to which the block is written to * @param baseStream optional. if non-null, <code>out</code> is assumed to * be a wrapper over this stream. This enables optimizations for * sending the data, e.g. * {@link SocketOutputStream#transferToFully(FileChannel, * long, int)}. * @param throttler for sending data. * @return total bytes reads, including crc. */ public long sendBlock(DataOutputStream out, OutputStream baseStream, DataTransferThrottler throttler) throws IOException { return sendBlock(out, baseStream, throttler, null); }
/** * Manage the OS buffer cache by performing read-ahead * and drop-behind. */ private void manageOsCache() throws IOException { // We can't manage the cache for this block if we don't have a file // descriptor to work with. if (ris.getDataInFd() == null) { return; } // Perform readahead if necessary if ((readaheadLength > 0) && (datanode.readaheadPool != null) && (alwaysReadahead || isLongRead())) { curReadahead = datanode.readaheadPool.readaheadStream( clientTraceFmt, ris.getDataInFd(), offset, readaheadLength, Long.MAX_VALUE, curReadahead); } // Drop what we've just read from cache, since we aren't // likely to need it again if (dropCacheBehindAllReads || (dropCacheBehindLargeReads && isLongRead())) { long nextCacheDropOffset = lastCacheDropOffset + CACHE_DROP_INTERVAL_BYTES; if (offset >= nextCacheDropOffset) { long dropLength = offset - lastCacheDropOffset; ris.dropCacheBehindReads(block.getBlockName(), lastCacheDropOffset, dropLength, POSIX_FADV_DONTNEED); lastCacheDropOffset = offset; } } }
/** * sendBlock() is used to read block and its metadata and stream the data to * either a client or to another datanode. * * @param out stream to which the block is written to * @param baseStream optional. if non-null, <code>out</code> is assumed to * be a wrapper over this stream. This enables optimizations for * sending the data, e.g. * {@link SocketOutputStream#transferToFully(FileChannel, * long, int)}. * @param throttler for sending data. * @return total bytes read, including checksum data. */ long sendBlock(DataOutputStream out, OutputStream baseStream, DataTransferThrottler throttler) throws IOException { final TraceScope scope = datanode.getTracer(). newScope("sendBlock_" + block.getBlockId()); try { return doSendBlock(out, baseStream, throttler); } finally { scope.close(); } }
(chunkSize * (long) maxChunks)); int numChunks = numberOfChunks(dataLen); // Number of chunks be sent in the packet int checksumDataLen = numChunks * checksumSize; int packetLen = dataLen + checksumDataLen + 4; int headerLen = writePacketHeader(pkt, dataLen, packetLen); readChecksum(buf, checksumOff, checksumDataLen); verifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff); throw ioeToSocketException(e);
HdfsConstants.SMALL_BUFFER_SIZE)); in = new DataInputStream(unbufIn); blockSender = new BlockSender(b, 0, b.getNumBytes(), false, false, true, DataNode.this, null, cachingStrategy); DatanodeInfo srcNode = new DatanodeInfo(bpReg); stage, 0, 0, 0, 0, blockSender.getChecksum(), cachingStrategy, false, false, null); blockSender.sendBlock(out, unbufOut, null);
blockSender = new BlockSender(namespaceId, block, startOffset, length, datanode.ignoreChecksumWhenRead, true, true, false, datanode, clientTraceFmt); long read = blockSender.sendBlock(out, baseStream, null); // send data out.flush(); if (blockSender.isBlockReadFully()) {
throw ioeToSocketException(e); long len = sendChunks(pktBuf, maxChunksPerPacket, streamForSendChunks); if (progress != null) { out.flush(); } catch (IOException e) { //socket error throw ioeToSocketException(e); ClientTraceLog.info(String.format(clientTraceFmt, totalRead, initialOffset, endTime - startTime)); close();
private void writeSuccessWithChecksumInfo(BlockSender blockSender, DataOutputStream out) throws IOException { ReadOpChecksumInfoProto ckInfo = ReadOpChecksumInfoProto.newBuilder() .setChecksum(DataTransferProtoUtil.toProto(blockSender.getChecksum())) .setChunkOffset(blockSender.getOffset()) .build(); BlockOpResponseProto response = BlockOpResponseProto.newBuilder() .setStatus(SUCCESS) .setReadOpChecksumInfo(ckInfo) .build(); response.writeDelimitedTo(out); out.flush(); }
final long replicaVisibleLength; synchronized(datanode.data) { replica = getReplica(block, datanode); replicaVisibleLength = replica.getVisibleLength(); if (replica instanceof FinalizedReplica) { waitForMinLength(rbw, startOffset + length); chunkChecksum = rbw.getLastChecksumAndDataLen();