Refine search
public void encode(OutputStream os) { try { CheckedOutputStream checkedOutputStream = new CheckedOutputStream(os, new CRC32()); encodeOrThrow(checkedOutputStream); long messageCrc = checkedOutputStream.getChecksum().getValue(); os.write((int) (0xFF & messageCrc >> 24)); os.write((int) (0xFF & messageCrc >> 16)); os.write((int) (0xFF & messageCrc >> 8)); os.write((int) (0xFF & messageCrc)); os.flush(); } catch (IOException ex) { throw new RuntimeException(ex); } }
long bytesSent = 0L; final Set<FlowFile> flowFilesSent = new HashSet<>(); final CRC32 crc = new CRC32(); logger.debug("{} Sending {} to {}", new Object[]{this, flowFile, peer}); final CheckedOutputStream checkedOutputStream = new CheckedOutputStream(flowFileOutputStream, crc); checkedOutputStream.close(); logger.debug("{} Sending FinishTransaction indicator to {}", this, peer); writeTransactionResponse(true, ResponseCode.FINISH_TRANSACTION, commsSession); calculatedCRC = String.valueOf(checkedOutputStream.getChecksum().getValue());
@Test public void testChecksumStreams() throws IOException { byte[] content = new byte[33333]; new Random().nextBytes(content); Murmur3F murmur3F = new Murmur3F(); murmur3F.update(content); String hash = murmur3F.getValueHexString(); murmur3F.reset(); CheckedOutputStream out = new CheckedOutputStream(new ByteArrayOutputStream(), murmur3F); out.write(content); Assert.assertEquals(hash, murmur3F.getValueHexString()); murmur3F.reset(); CheckedInputStream in = new CheckedInputStream(new ByteArrayInputStream(content), murmur3F); IoUtils.readAllBytes(in); Assert.assertEquals(hash, murmur3F.getValueHexString()); }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into * @param fsync sync the file immediately after write */ public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot, boolean fsync) throws IOException { if (!close) { try (CheckedOutputStream crcOut = new CheckedOutputStream(new BufferedOutputStream(fsync ? new AtomicFileOutputStream(snapShot) : new FileOutputStream(snapShot)), new Adler32())) { //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt, sessions, oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); crcOut.flush(); } } }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into */ public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot) throws IOException { if (!close) { OutputStream sessOS = new BufferedOutputStream(new FileOutputStream(snapShot)); CheckedOutputStream crcOut = new CheckedOutputStream(sessOS, new Adler32()); //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt,sessions,oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); sessOS.flush(); crcOut.close(); sessOS.close(); } }
private static int crc32(BytesReference data) { OutputStream dummy = new OutputStream() { @Override public void write(int b) throws IOException { // no-op } @Override public void write(byte[] b, int off, int len) throws IOException { // no-op } }; CRC32 crc32 = new CRC32(); try { data.writeTo(new CheckedOutputStream(dummy, crc32)); } catch (IOException bogus) { // cannot happen throw new Error(bogus); } return (int) crc32.getValue(); }
searchForDeltas(compressMonitor); crc32 = new CRC32(); final PackOutputStream out = new PackOutputStream( writeMonitor, isIndexDisabled() ? packStream : new CheckedOutputStream(packStream, crc32), this);
/** * zip * * @throws java.io.IOException if any. */ public void zip() throws IOException { zos = new ZipOutputStream(target); cos = new CheckedOutputStream(zos, new CRC32()); zos.setComment(comment); for(String name:multiInputStream.keySet()){ ZipEntry ze = new ZipEntry(name); zos.putNextEntry(ze); InputStream is = multiInputStream.get(name); is = new BufferedInputStream(is); int readSize = -1; byte[] temp = new byte[BUFFER_SIZE]; while((readSize = is.read(temp, 0 , BUFFER_SIZE)) != -1){ cos.write(temp, 0, readSize); } } }
ByteStreams.copy(is, jarOut); } else { crc32.reset(); TransferByteOutputStream os = new TransferByteOutputStream(); CheckedOutputStream checkedOut = new CheckedOutputStream(os, crc32); ByteStreams.copy(is, checkedOut); checkedOut.close(); jarEntry.setCrc(checkedOut.getChecksum().getValue()); jarOut.putNextEntry(jarEntry); os.transfer(jarOut);
public void writeToFile(Path loc, JobConf job, Checksum crc) throws IOException { final FileSystem rfs = FileSystem.getLocal(job).getRaw(); CheckedOutputStream chk = null; final FSDataOutputStream out = rfs.create(loc); try { if (crc != null) { crc.reset(); chk = new CheckedOutputStream(out, crc); chk.write(buf.array()); out.writeLong(chk.getChecksum().getValue()); } else { out.write(buf.array()); } } finally { if (chk != null) { chk.close(); } else { out.close(); } } }
private ChunkBuffer (ByteArrayOutputStream buffer, CRC32 crc) { super(new CheckedOutputStream(buffer, crc)); this.buffer = buffer; this.crc = crc; }
private void flushBuffer() throws IOException { final UnsafeByteArrayOutputStream compressedBuffer = new UnsafeByteArrayOutputStream(blockSize+4*numRecords); final CheckedOutputStream checksumStream = new CheckedOutputStream(compressedBuffer, new Adler32()); final DataOutputStream compressorStream = new DataOutputStream(codec.createOutputStream(checksumStream)); compressorStream.writeInt(numRecords); for (int i = 0; i < numRecords; i++) { VIntUtils.writeVInt((OutputStream)compressorStream, lengthBuffer[i]); } compressorStream.write(currentBlockBytes.getByteArray(), 0, currentBlockBytes.size()); compressorStream.close(); out.writeInt(compressedBuffer.size()); final int checksum = (int)checksumStream.getChecksum().getValue(); out.writeInt(checksum); out.write(compressedBuffer.getByteArray(), 0, compressedBuffer.size()); currentBlockBytes.reset(); numRecords = 0; final int padLength = (int)(pad-out.position()%pad); if (padLength != pad) { for (int i = 0; i < padLength; i++) { out.writeByte(0); } } blockAddress = out.position()<<shift; }
protected static void put(CheckedOutputStream out, char c) throws IOException { out.write((byte)c); }
public Checksum getChecksum(){ return os.getChecksum(); }
public void close() throws IOException { os.close(); } }
public void end() throws IOException { out.write(END_OF_STREAM); out.write(toByteArray(out.getChecksum().getValue())); out.flush(); } }
/** * Create a {@link CompressedXContent} out of a {@link ToXContent} instance. */ public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); OutputStream compressedStream = CompressorFactory.COMPRESSOR.streamOutput(bStream); CRC32 crc32 = new CRC32(); OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32); try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) { builder.startObject(); xcontent.toXContent(builder, params); builder.endObject(); } this.bytes = BytesReference.toBytes(bStream.bytes()); this.crc32 = (int) crc32.getValue(); assertConsistent(); }
/** * 将OutputStream 转换为 ZipOutputStream 并作为 compressFile 的输入参数 * 这个可以用于 打包下载 * @param os os * @return ZipOutputStream */ public static ZipOutputStream convertToZipOutputStream(OutputStream os){ return new ZipOutputStream( new CheckedOutputStream(os, new CRC32())); }
ByteStreams.copy(is, jarOut); } else { crc32.reset(); TransferByteOutputStream os = new TransferByteOutputStream(); CheckedOutputStream checkedOut = new CheckedOutputStream(os, crc32); ByteStreams.copy(is, checkedOut); checkedOut.close(); jarEntry.setCrc(checkedOut.getChecksum().getValue()); jarOut.putNextEntry(jarEntry); os.transfer(jarOut);
public void writeToFile(Path loc, JobConf job, Checksum crc) throws IOException { final FileSystem rfs = FileSystem.getLocal(job).getRaw(); CheckedOutputStream chk = null; final FSDataOutputStream out = rfs.create(loc); try { if (crc != null) { crc.reset(); chk = new CheckedOutputStream(out, crc); chk.write(buf.array()); out.writeLong(chk.getChecksum().getValue()); } else { out.write(buf.array()); } } finally { if (chk != null) { chk.close(); } else { out.close(); } } }