@Override public void flush() throws IOException { // Make sure we flush out our stream out.flush(); // Have to finish the deflater in order to get proper block. dout.finish(); // Grab size of compressed data, write this out and then the block // of compressed data. Set number of bytes written to this value. int compressedSize = buffer.size(); written = compressedSize; written += NcStream.writeVInt(writer, compressedSize); buffer.writeTo(writer); // Reset buffer so that in theory we could continue to write to this // stream. buffer.reset(); } }
public OutputStream setupStream(OutputStream out, int size) throws IOException { switch (type) { // For compression (currently deflate) we compress the data, then // will write the block size, and then data, when the stream is closed. case DEFLATE: // limit level to range [-1, 9], where -1 is default deflate setting. int level = Math.min(Math.max((Integer)compressInfo, -1), 9); int bufferSize = Math.min(size / 2, 512 * 1024 * 1024); return new NcStreamCompressedOutputStream(out, bufferSize, level); default: System.out.printf(" Unknown compression type %s. Defaulting to none.%n", type); // In the case of no compression, go ahead and write the block // size so that the stream is ready for data case NONE: NcStream.writeVInt(out, size); return out; } } }
public long sendHeader(WritableByteChannel wbc) throws IOException { long size = 0; //// header message size += writeBytes(wbc, NcStream.MAGIC_HEADER); byte[] b = header.toByteArray(); size += NcStream.writeVInt(wbc, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += writeBytes(wbc, b); if (show) System.out.println(" header size=" + size); return size; }
public long sendHeader(WritableByteChannel wbc) throws IOException { long size = 0; //// header message size += writeBytes(wbc, NcStream.MAGIC_HEADER); byte[] b = header.toByteArray(); size += NcStream.writeVInt(wbc, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += writeBytes(wbc, b); if (show) System.out.println(" header size=" + size); return size; }
public long sendHeader(OutputStream out) throws IOException { long size = 0; //// header message size += writeBytes(out, NcStream.MAGIC_HEADER); byte[] b = header.toByteArray(); size += NcStream.writeVInt(out, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += writeBytes(out, b); if (show) System.out.println(" header size=" + size); return size; }
public long sendHeader(WritableByteChannel wbc) throws IOException { long size = 0; //// header message size += writeBytes(wbc, NcStream.MAGIC_HEADER); byte[] b = header.toByteArray(); size += NcStream.writeVInt(wbc, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += writeBytes(wbc, b); if (show) System.out.println(" header size=" + size); return size; }
public long sendHeader(OutputStream out) throws IOException { long size = 0; //// header message size += writeBytes(out, NcStream.MAGIC_HEADER); byte[] b = header.toByteArray(); size += NcStream.writeVInt(out, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += writeBytes(out, b); if (show) System.out.println(" header size=" + size); return size; }
public long sendHeader(OutputStream out) throws IOException { long size = 0; //// header message size += writeBytes(out, NcStream.MAGIC_HEADER); byte[] b = header.toByteArray(); size += NcStream.writeVInt(out, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += writeBytes(out, b); if (show) System.out.println(" header size=" + size); return size; }
private long sendData(Array data, OutputStream out, boolean deflate) throws IOException, InvalidRangeException { // length of data uncompressed long uncompressedLength = data.getSizeBytes(); long size = 0; if (deflate) { // write to an internal buffer, so we can find out the size ByteArrayOutputStream bout = new ByteArrayOutputStream(); DeflaterOutputStream dout = new DeflaterOutputStream(bout); IospHelper.copyToOutputStream(data, dout); // write internal buffer to output stream dout.close(); int deflatedSize = bout.size(); size += NcStream.writeVInt(out, deflatedSize); bout.writeTo(out); size += deflatedSize; } else { size += NcStream.writeVInt(out, (int) uncompressedLength); size += IospHelper.copyToOutputStream(data, out); } return size; }
public static long encodeArrayStructure(ArrayStructure as, OutputStream os) throws java.io.IOException { long size = 0; ArrayStructureBB dataBB = StructureDataDeep.copyToArrayBB(as, null); // LOOK what should the bo be ?? List<String> ss = new ArrayList<String>(); List<Object> heap = dataBB.getHeap(); List<Integer> count = new ArrayList<Integer>(); if (heap != null) { for (Object ho : heap) { if (ho instanceof String) { count.add(1); ss.add((String) ho); } else if (ho instanceof String[]) { String[] hos = (String[]) ho; count.add(hos.length); for (String s : hos) ss.add(s); } } } ByteBuffer bb = dataBB.getByteBuffer(); NcStreamProto.StructureData proto = NcStream.encodeStructureDataProto(bb.array(), count, ss); byte[] datab = proto.toByteArray(); size += NcStream.writeVInt(os, datab.length); // proto len os.write(datab); // proto size += datab.length; // System.out.printf("encodeArrayStructure write sdata size= %d%n", datab.length); return size; }
public static long encodeArrayStructure(ArrayStructure as, OutputStream os) throws java.io.IOException { long size = 0; ArrayStructureBB dataBB = IospHelper.makeArrayBB(as); List<String> ss = new ArrayList<String>(); List<Object> heap = dataBB.getHeap(); List<Integer> count = new ArrayList<Integer>(); if (heap != null) { for (Object ho : heap) { if (ho instanceof String) { count.add(1); ss.add((String) ho); } else if (ho instanceof String[]) { String[] hos = (String[]) ho; count.add(hos.length); for (String s : hos) ss.add(s); } } } ByteBuffer bb = dataBB.getByteBuffer(); NcStreamProto.StructureData proto = NcStream.encodeStructureDataProto(bb.array(), count, ss); byte[] datab = proto.toByteArray(); size += NcStream.writeVInt(os, datab.length); // proto len os.write(datab); // proto size += datab.length; // System.out.printf("encodeArrayStructure write sdata size= %d%n", datab.length); return size; }
public static long encodeArrayStructure(ArrayStructure as, ByteOrder bo, OutputStream os) throws java.io.IOException { long size = 0; ArrayStructureBB dataBB = StructureDataDeep.copyToArrayBB(as, bo, true); // force canonical packing List<String> ss = new ArrayList<>(); List<Object> heap = dataBB.getHeap(); List<Integer> count = new ArrayList<>(); if (heap != null) { for (Object ho : heap) { if (ho instanceof String) { count.add(1); ss.add((String) ho); } else if (ho instanceof String[]) { String[] hos = (String[]) ho; count.add(hos.length); for (String s : hos) ss.add(s); } } } // LOOK optionally compress StructureMembers sm = dataBB.getStructureMembers(); ByteBuffer bb = dataBB.getByteBuffer(); NcStreamProto.StructureData proto = NcStream.encodeStructureDataProto(bb.array(), count, ss, (int) as.getSize(), sm.getStructureSize()); byte[] datab = proto.toByteArray(); size += NcStream.writeVInt(os, datab.length); // proto len os.write(datab); // proto size += datab.length; // System.out.printf("encodeArrayStructure write sdata size= %d%n", datab.length); return size; }
public long sendHeader(OutputStream out, CoverageCollection gridDataset, String location) throws IOException { long size = 0; CdmrFeatureProto.CoverageDataset.Builder headerBuilder = encodeHeader(gridDataset, location); CdmrFeatureProto.CoverageDataset header = headerBuilder.build(); // header message size += NcStream.writeBytes(out, NcStream.MAGIC_HEADERCOV); byte[] b = header.toByteArray(); size += NcStream.writeVInt(out, b.length); // len if (show) System.out.println("Write Header len=" + b.length); // payload size += NcStream.writeBytes(out, b); if (show) System.out.println(" header size=" + size); return size; }
NcStream.writeVInt(raf, b.length); // message size
NcStream.writeVInt(raf, b.length); // message size
public long sendData2(Variable v, Section section, OutputStream out, NcStreamCompression compress) throws IOException, InvalidRangeException { if (show) System.out.printf(" %s section=%s%n", v.getFullName(), section); boolean isVlen = v.isVariableLength(); // && v.getRank() > 1; if (isVlen) v.read(section); NcStreamDataCol encoder = new NcStreamDataCol(); NcStreamProto.DataCol dataProto = encoder.encodeData2(v.getFullName(), isVlen, section, v.read(section)); // LOOK trap error, write error message ?? // dataProto.writeDelimitedTo(out); long size = 0; size += writeBytes(out, NcStream.MAGIC_DATA2); // data version 3 byte[] datab = dataProto.toByteArray(); size += NcStream.writeVInt(out, datab.length); // dataProto len size += writeBytes(out, datab); // dataProto return size; }