crcOut.getChannel().position(crcDiskSize); return new ReplicaOutputStreams(blockOut, crcOut, checksum, getVolume(), fileIoProvider); } catch (IOException e) {
/** * Sets the offset in the meta file so that the * last checksum will be overwritten. */ @Override // FsDatasetSpi public void adjustCrcChannelPosition(ExtendedBlock b, ReplicaOutputStreams streams, int checksumSize) throws IOException { FileOutputStream file = (FileOutputStream)streams.getChecksumOut(); FileChannel channel = file.getChannel(); long oldPos = channel.position(); long newPos = oldPos - checksumSize; if (LOG.isDebugEnabled()) { LOG.debug("Changing meta file offset of block " + b + " from " + oldPos + " to " + newPos); } channel.position(newPos); }
if (syncOnClose && (streams.getDataOut() != null || checksumOut != null)) { datanode.metrics.incrFsyncCount(); if (syncOnClose) { long fsyncStartNanos = flushEndNanos; streams.syncChecksumOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos); if (streams.getDataOut() != null) { long flushStartNanos = System.nanoTime(); streams.flushDataOut(); long flushEndNanos = System.nanoTime(); if (syncOnClose) { long fsyncStartNanos = flushEndNanos; streams.syncDataOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos); streams.closeDataStream(); streams.close();
long flushEndNanos = System.nanoTime(); if (isSync) { streams.syncChecksumOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - flushEndNanos); if (streams.getDataOut() != null) { long flushStartNanos = System.nanoTime(); streams.flushDataOut(); long flushEndNanos = System.nanoTime(); if (isSync) { long fsyncStartNanos = flushEndNanos; streams.syncDataOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos); dirSyncOnHSyncDone = true; if (checksumOut != null || streams.getDataOut() != null) { datanode.metrics.addFlushNanos(flushTotalNanos); if (isSync) {
if (checksumReceivedLen == 0 && !streams.isTransientStorage()) { && streams.isTransientStorage(); try { long onDiskLen = replicaInfo.getBytesOnDisk(); streams.writeDataToDisk(dataBuf.array(), startByteToDisk, numBytesToDisk); long duration = Time.monotonicNow() - begin;
this.diskChecksum = streams.getChecksum(); this.needsChecksumTranslation = !clientChecksum.equals(diskChecksum); this.bytesPerChecksum = diskChecksum.getBytesPerChecksum(); this.checksumSize = diskChecksum.getChecksumSize(); this.out = streams.getDataOut(); if (out instanceof FileOutputStream) { this.outFd = ((FileOutputStream)out).getFD(); streams.getChecksumOut(), HdfsConstants.SMALL_BUFFER_SIZE));
if (isSync) { long fsyncStartNanos = flushEndNanos; streams.syncChecksumOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos); if (isSync) { long fsyncStartNanos = flushEndNanos; streams.syncDataOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
this.diskChecksum = streams.getChecksum(); this.needsChecksumTranslation = !clientChecksum.equals(diskChecksum); this.bytesPerChecksum = diskChecksum.getBytesPerChecksum(); streams.getChecksumOut(), DFSUtilClient.getSmallBufferSize( datanode.getConf())));
streams = replicaInfo.createStreams(true, DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); streams.getChecksumOut().write('a'); dn.data.initReplicaRecovery(new RecoveringBlock(block, null, RECOVERY_ID+1)); try { anyBoolean(), any(DatanodeID[].class), any(String[].class)); } finally { streams.close();
int addSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId) throws IOException { int bytesAdded = 0; for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) { ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual // data written ReplicaInPipelineInterface bInfo = fsdataset.createRbw( StorageType.DEFAULT, b, false).getReplica(); ReplicaOutputStreams out = bInfo.createStreams(true, DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); try { OutputStream dataOut = out.getDataOut(); assertEquals(0, fsdataset.getLength(b)); for (int j=1; j <= blockIdToLen(i); ++j) { dataOut.write(j); assertEquals(j, bInfo.getBytesOnDisk()); // correct length even as we write bytesAdded++; } } finally { out.close(); } b.setNumBytes(blockIdToLen(i)); fsdataset.finalizeBlock(b, false); assertEquals(blockIdToLen(i), fsdataset.getLength(b)); } return bytesAdded; } int addSomeBlocks(SimulatedFSDataset fsdataset ) throws IOException {
IOUtils.closeStream(streams.getDataOut());
if (checksumReceivedLen == 0 && !streams.isTransientStorage()) { && streams.isTransientStorage(); try { long onDiskLen = replicaInfo.getBytesOnDisk();
this.diskChecksum = streams.getChecksum(); this.needsChecksumTranslation = !clientChecksum.equals(diskChecksum); this.bytesPerChecksum = diskChecksum.getBytesPerChecksum(); this.checksumSize = diskChecksum.getChecksumSize(); this.out = streams.getDataOut(); if (out instanceof FileOutputStream) { this.outFd = ((FileOutputStream)out).getFD(); streams.getChecksumOut(), HdfsConstants.SMALL_BUFFER_SIZE));
if (isSync) { long fsyncStartNanos = flushEndNanos; streams.syncChecksumOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos); if (isSync) { long fsyncStartNanos = flushEndNanos; streams.syncDataOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
ReplicaOutputStreams outputStreams = rbw.createStreams(false, DEFAULT_CHECKSUM); OutputStream dataOutput = outputStreams.getDataOut();
if (checksumReceivedLen == 0 && !streams.isTransientStorage()) { && streams.isTransientStorage(); try { long onDiskLen = replicaInfo.getBytesOnDisk();
@Override public ReplicaOutputStreams createStreams(boolean isCreate, DataChecksum requestedChecksum) throws IOException { return new ReplicaOutputStreams(null, null, requestedChecksum, false); }
if (syncOnClose) { long fsyncStartNanos = flushEndNanos; streams.syncChecksumOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos); if (syncOnClose) { long fsyncStartNanos = flushEndNanos; streams.syncDataOut(); datanode.metrics.addFsyncNanos(System.nanoTime() - fsyncStartNanos);
/** * Sets the offset in the meta file so that the * last checksum will be overwritten. */ @Override // FsDatasetSpi public void adjustCrcChannelPosition(ExtendedBlock b, ReplicaOutputStreams streams, int checksumSize) throws IOException { FileOutputStream file = (FileOutputStream)streams.getChecksumOut(); FileChannel channel = file.getChannel(); long oldPos = channel.position(); long newPos = oldPos - checksumSize; if (LOG.isDebugEnabled()) { LOG.debug("Changing meta file offset of block " + b + " from " + oldPos + " to " + newPos); } channel.position(newPos); }
@Override synchronized public ReplicaOutputStreams createStreams(boolean isCreate, DataChecksum requestedChecksum) throws IOException { if (finalized) { throw new IOException("Trying to write to a finalized replica " + theBlock); } else { SimulatedOutputStream crcStream = new SimulatedOutputStream(); return new ReplicaOutputStreams(oStream, crcStream, requestedChecksum, getStorage(theBlock).getVolume().isTransientStorage()); } }