private ChunkBuffer (ByteArrayOutputStream buffer, CRC32 crc) { super(new CheckedOutputStream(buffer, crc)); this.buffer = buffer; this.crc = crc; }
private ChunkBuffer (ByteArrayOutputStream buffer, CRC32 crc) { super(new CheckedOutputStream(buffer, crc)); this.buffer = buffer; this.crc = crc; }
/** * Creates a new {@link OutputStreamIndexOutput} with the given buffer size. * @param bufferSize the buffer size in bytes used to buffer writes internally. * @throws IllegalArgumentException if the given buffer size is less or equal to <tt>0</tt> */ public OutputStreamIndexOutput(String resourceDescription, String name, OutputStream out, int bufferSize) { super(resourceDescription, name); this.os = new BufferedOutputStream(new CheckedOutputStream(out, crc), bufferSize); }
public void encode(OutputStream os) { try { CheckedOutputStream checkedOutputStream = new CheckedOutputStream(os, new CRC32()); encodeOrThrow(checkedOutputStream); long messageCrc = checkedOutputStream.getChecksum().getValue(); os.write((int) (0xFF & messageCrc >> 24)); os.write((int) (0xFF & messageCrc >> 16)); os.write((int) (0xFF & messageCrc >> 8)); os.write((int) (0xFF & messageCrc)); os.flush(); } catch (IOException ex) { throw new RuntimeException(ex); } }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into * @param fsync sync the file immediately after write */ public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot, boolean fsync) throws IOException { if (!close) { try (CheckedOutputStream crcOut = new CheckedOutputStream(new BufferedOutputStream(fsync ? new AtomicFileOutputStream(snapShot) : new FileOutputStream(snapShot)), new Adler32())) { //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt, sessions, oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); crcOut.flush(); } } }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into */ public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot) throws IOException { if (!close) { OutputStream sessOS = new BufferedOutputStream(new FileOutputStream(snapShot)); CheckedOutputStream crcOut = new CheckedOutputStream(sessOS, new Adler32()); //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt,sessions,oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); sessOS.flush(); crcOut.close(); sessOS.close(); } }
final OutputStream out = new CheckedOutputStream(dataOut, crc);
logger.debug("{} Sending {} to {}", new Object[]{this, flowFile, peer}); final CheckedOutputStream checkedOutputStream = new CheckedOutputStream(flowFileOutputStream, crc);
@Test public void testChecksumStreams() throws IOException { byte[] content = new byte[33333]; new Random().nextBytes(content); Murmur3F murmur3F = new Murmur3F(); murmur3F.update(content); String hash = murmur3F.getValueHexString(); murmur3F.reset(); CheckedOutputStream out = new CheckedOutputStream(new ByteArrayOutputStream(), murmur3F); out.write(content); Assert.assertEquals(hash, murmur3F.getValueHexString()); murmur3F.reset(); CheckedInputStream in = new CheckedInputStream(new ByteArrayInputStream(content), murmur3F); IoUtils.readAllBytes(in); Assert.assertEquals(hash, murmur3F.getValueHexString()); }
private static int crc32(BytesReference data) { OutputStream dummy = new OutputStream() { @Override public void write(int b) throws IOException { // no-op } @Override public void write(byte[] b, int off, int len) throws IOException { // no-op } }; CRC32 crc32 = new CRC32(); try { data.writeTo(new CheckedOutputStream(dummy, crc32)); } catch (IOException bogus) { // cannot happen throw new Error(bogus); } return (int) crc32.getValue(); }
/** * Create a {@link CompressedXContent} out of a {@link ToXContent} instance. */ public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); OutputStream compressedStream = CompressorFactory.COMPRESSOR.streamOutput(bStream); CRC32 crc32 = new CRC32(); OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32); try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) { builder.startObject(); xcontent.toXContent(builder, params); builder.endObject(); } this.bytes = BytesReference.toBytes(bStream.bytes()); this.crc32 = (int) crc32.getValue(); assertConsistent(); }
private static void writeFile(FileSystem fs, Path f, long fill, int parts) throws IOException { FSDataOutputStream out = fs.create(f, false); CheckedOutputStream iout = new CheckedOutputStream(out, new CRC32()); DataOutputStream dout = new DataOutputStream(iout); for (int i = 0; i < parts; ++i) { for (int j = 0; j < MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; ++j) { dout.writeLong(fill); } } out.writeLong(iout.getChecksum().getValue()); dout.close(); } }
CheckedOutputStream iout = new CheckedOutputStream(out, new CRC32()); DataOutputStream dout = new DataOutputStream(iout); for (int i = 0; i < parts; ++i) {
private ChunkBuffer (ByteArrayOutputStream buffer, CRC32 crc) { super(new CheckedOutputStream(buffer, crc)); this.buffer = buffer; this.crc = crc; }
private ChunkBuffer (ByteArrayOutputStream buffer, CRC32 crc) { super(new CheckedOutputStream(buffer, crc)); this.buffer = buffer; this.crc = crc; }
private ChunkBuffer (ByteArrayOutputStream buffer, CRC32 crc) { super(new CheckedOutputStream(buffer, crc)); this.buffer = buffer; this.crc = crc; }
isIndexDisabled() ? packStream : new CheckedOutputStream(packStream, crc32), this);
/** * Creates a new {@link OutputStreamIndexOutput} with the given buffer size. * @param bufferSize the buffer size in bytes used to buffer writes internally. * @throws IllegalArgumentException if the given buffer size is less or equal to <tt>0</tt> */ public OutputStreamIndexOutput(String resourceDescription, String name, OutputStream out, int bufferSize) { super(resourceDescription, name); this.os = new BufferedOutputStream(new CheckedOutputStream(out, crc), bufferSize); }
/** * Creates a new {@link OutputStreamIndexOutput} with the given buffer size. * @param bufferSize the buffer size in bytes used to buffer writes internally. * @throws IllegalArgumentException if the given buffer size is less or equal to <tt>0</tt> */ public OutputStreamIndexOutput(String resourceDescription, OutputStream out, int bufferSize) { super(resourceDescription); this.os = new BufferedOutputStream(new CheckedOutputStream(out, crc), bufferSize); }
/** * 将OutputStream 转换为 ZipOutputStream 并作为 compressFile 的输入参数 * 这个可以用于 打包下载 * @param os os * @return ZipOutputStream */ public static ZipOutputStream convertToZipOutputStream(OutputStream os){ return new ZipOutputStream( new CheckedOutputStream(os, new CRC32())); }