ReplicaInputStreams ris = new ReplicaInputStreams(blockIn, checksumIn, volume.obtainReference(), fileIoProvider)) { ris.skipChecksumFully((numChunks - 1) * checksumSize); long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum; ris.skipDataFully(lastChunkStartPos); int lastChunkSize = (int) Math.min( bytesPerChecksum, blockFileLen - lastChunkStartPos); byte[] buf = new byte[lastChunkSize + checksumSize]; ris.readChecksumFully(buf, lastChunkSize, checksumSize); ris.readDataFully(buf, 0, lastChunkSize); checksum.update(buf, 0, lastChunkSize); long validFileLength;
byte[] buf = pkt.array(); if (checksumSize > 0 && ris.getChecksumIn() != null) { readChecksum(buf, checksumOff, checksumDataLen); ris.readDataFully(buf, dataOff, dataLen); FileChannel fileCh = ((FileInputStream)ris.getDataIn()).getChannel(); LongWritable waitTime = new LongWritable(); LongWritable transferTime = new LongWritable(); fileIoProvider.transferToSocketFully( ris.getVolumeRef().getVolume(), sockOut, fileCh, blockInPosition, dataLen, waitTime, transferTime); datanode.metrics.addSendDataPacketBlockedOnNetworkNanos(waitTime.get()); LOG.error("BlockSender.sendChunks() exception: ", e); datanode.getBlockScanner().markSuspectBlock( ris.getVolumeRef().getVolume().getStorageID(), block);
/** * Read checksum into given buffer * @param buf buffer to read the checksum into * @param checksumOffset offset at which to write the checksum into buf * @param checksumLen length of checksum to write * @throws IOException on error */ private void readChecksum(byte[] buf, final int checksumOffset, final int checksumLen) throws IOException { if (checksumSize <= 0 && ris.getChecksumIn() == null) { return; } try { ris.readChecksumFully(buf, checksumOffset, checksumLen); } catch (IOException e) { LOG.warn(" Could not read or failed to verify checksum for data" + " at offset " + offset + " for block " + block, e); ris.closeChecksumStream(); if (corruptChecksumOk) { if (checksumLen > 0) { // Just fill the array with zeros. Arrays.fill(buf, checksumOffset, checksumOffset + checksumLen, (byte) 0); } } else { throw e; } } }
/** * close opened files. */ @Override public void close() throws IOException { if (ris.getDataInFd() != null && ((dropCacheBehindAllReads) || (dropCacheBehindLargeReads && isLongRead()))) { try { ris.dropCacheBehindReads(block.getBlockName(), lastCacheDropOffset, offset - lastCacheDropOffset, POSIX_FADV_DONTNEED); } catch (Exception e) { LOG.warn("Unable to drop cache on file close", e); } } if (curReadahead != null) { curReadahead.cancel(); } try { ris.closeStreams(); } finally { IOUtils.closeStream(ris); ris = null; } }
/** * Manage the OS buffer cache by performing read-ahead * and drop-behind. */ private void manageOsCache() throws IOException { // We can't manage the cache for this block if we don't have a file // descriptor to work with. if (ris.getDataInFd() == null) { return; } // Perform readahead if necessary if ((readaheadLength > 0) && (datanode.readaheadPool != null) && (alwaysReadahead || isLongRead())) { curReadahead = datanode.readaheadPool.readaheadStream( clientTraceFmt, ris.getDataInFd(), offset, readaheadLength, Long.MAX_VALUE, curReadahead); } // Drop what we've just read from cache, since we aren't // likely to need it again if (dropCacheBehindAllReads || (dropCacheBehindLargeReads && isLongRead())) { long nextCacheDropOffset = lastCacheDropOffset + CACHE_DROP_INTERVAL_BYTES; if (offset >= nextCacheDropOffset) { long dropLength = offset - lastCacheDropOffset; ris.dropCacheBehindReads(block.getBlockName(), lastCacheDropOffset, dropLength, POSIX_FADV_DONTNEED); lastCacheDropOffset = offset; } } }
ris = new ReplicaInputStreams( blockIn, checksumIn, volumeRef, fileIoProvider); } catch (IOException ioe) {
try (ReplicaInputStreams instr = datanode.data.getTmpInputStreams(block, blkoff, ckoff)) { IOUtils.readFully(instr.getDataIn(), buf, 0, sizePartialChunk); IOUtils.readFully(instr.getChecksumIn(), crcbuf, 0, crcbuf.length);
if (isLongRead() && ris.getDataInFd() != null) { ris.dropCacheBehindReads(block.getBlockName(), 0, 0, POSIX_FADV_SEQUENTIAL); boolean transferTo = transferToAllowed && !verifyChecksum && baseStream instanceof SocketOutputStream && ris.getDataIn() instanceof FileInputStream; if (transferTo) { FileChannel fileChannel = ((FileInputStream)ris.getDataIn()).getChannel(); blockInPosition = fileChannel.position(); streamForSendChunks = baseStream;
/** * Returns handles to the block file and its metadata file */ @Override // FsDatasetSpi public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkOffset, long metaOffset) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { ReplicaInfo info = getReplicaInfo(b); FsVolumeReference ref = info.getVolume().obtainReference(); try { InputStream blockInStream = info.getDataInputStream(blkOffset); try { InputStream metaInStream = info.getMetadataInputStream(metaOffset); return new ReplicaInputStreams( blockInStream, metaInStream, ref, datanode.getFileIoProvider()); } catch (IOException e) { IOUtils.cleanup(null, blockInStream); throw e; } } catch (IOException e) { IOUtils.cleanup(null, ref); throw e; } } }
try (ReplicaInputStreams instr = datanode.data.getTmpInputStreams(block, blkoff, ckoff)) { IOUtils.readFully(instr.getDataIn(), buf, 0, sizePartialChunk); IOUtils.readFully(instr.getChecksumIn(), crcbuf, 0, crcbuf.length);
@Override public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkoff, long ckoff) throws IOException { return new ReplicaInputStreams(null, null, null); }
/** * Returns handles to the block file and its metadata file */ @Override // FsDatasetSpi public synchronized ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkOffset, long metaOffset) throws IOException { ReplicaInfo info = getReplicaInfo(b); FsVolumeReference ref = info.getVolume().obtainReference(); try { InputStream blockInStream = openAndSeek(info.getBlockFile(), blkOffset); try { InputStream metaInStream = openAndSeek(info.getMetaFile(), metaOffset); return new ReplicaInputStreams(blockInStream, metaInStream, ref); } catch (IOException e) { IOUtils.cleanup(null, blockInStream); throw e; } } catch (IOException e) { IOUtils.cleanup(null, ref); throw e; } }
/** * Returns handles to the block file and its metadata file */ @Override // FsDatasetSpi public synchronized ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkOffset, long metaOffset) throws IOException { ReplicaInfo info = getReplicaInfo(b); FsVolumeReference ref = info.getVolume().obtainReference(); try { InputStream blockInStream = openAndSeek(info.getBlockFile(), blkOffset); try { InputStream metaInStream = openAndSeek(info.getMetaFile(), metaOffset); return new ReplicaInputStreams(blockInStream, metaInStream, ref); } catch (IOException e) { IOUtils.cleanup(null, blockInStream); throw e; } } catch (IOException e) { IOUtils.cleanup(null, ref); throw e; } }