Coverage decodeGrid(CdmrFeatureProto.Coverage proto, CoverageReader reader) { DataType dataType = NcStream.convertDataType(proto.getDataType()); List<Attribute> atts = new ArrayList<>(); for (ucar.nc2.stream.NcStreamProto.Attribute patt : proto.getAttsList()) atts.add(NcStream.decodeAtt(patt)); return new Coverage(proto.getName(), dataType, atts, proto.getCoordSys(), proto.getUnits(), proto.getDescription(), reader, null); }
public NcsMess(long filePos, int len, Object what) { this.filePos = filePos; this.len = len; this.what = what; if (what instanceof NcStreamProto.Data) { NcStreamProto.Data dataMess = (NcStreamProto.Data) what; this.dataType = NcStream.convertDataType(dataMess.getDataType()); this.varName = dataMess.getVarName(); Section s = NcStream.decodeSection(dataMess.getSection()); if (s != null) this.nelems = (int) s.computeSize(); } else if (what instanceof DataStorage) this.nelems = ((DataStorage) what).nelems; }
static Variable decodeVar(NetcdfFile ncfile, Group g, Structure parent, NcStreamProto.Variable var) { Variable ncvar = new Variable(ncfile, g, parent, var.getName()); DataType varType = convertDataType(var.getDataType()); ncvar.setDataType(convertDataType(var.getDataType()));
static public NcStreamProto.Data encodeDataProto(String varname, DataType datatype, Section section, boolean deflate, int uncompressedLength) { NcStreamProto.Data.Builder builder = NcStreamProto.Data.newBuilder(); builder.setVarName(varname); builder.setDataType(convertDataType(datatype)); builder.setSection(encodeSection(section)); if (deflate) { builder.setCompress(NcStreamProto.Compress.DEFLATE); builder.setUncompressedSize(uncompressedLength); } builder.setVersion(ncstream_data_version); return builder.build(); }
ProtobufPointFeatureMaker(PointStreamProto.PointFeatureCollection pfc) throws IOException { try { // LOOK No calendar dateUnit = CalendarDateUnit.of(null, pfc.getTimeUnit()); } catch (Exception e) { e.printStackTrace(); dateUnit = CalendarDateUnit.unixDateUnit; } sm = new StructureMembers(pfc.getName()); for (PointStreamProto.Member m : pfc.getMembersList()) { String name = m.getName(); String desc = m.getDesc().length() > 0 ? m.getDesc() : null; String units = m.getUnits().length() > 0 ? m.getUnits() : null; DataType dtype = NcStream.convertDataType(m.getDataType()); int[] shape = NcStream.decodeSection(m.getSection()).getShape(); sm.addMember(name, desc, units, dtype, shape); } ArrayStructureBB.setOffsets(sm); }
public GeoReferencedArray decodeGeoReferencedArray(CoverageDataResponse dataResponse, CdmrFeatureProto.GeoReferencedArray parray) { DataType dataType = NcStream.convertDataType(parray.getDataType()); ByteOrder byteOrder = parray.getBigend() ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; boolean deflate = parray.getCompress() == NcStreamProto.Compress.DEFLATE; long uncompressedSize = parray.getUncompressedSize(); int[] shape = new int[parray.getShapeCount()]; for (int i=0; i< parray.getShapeCount(); i++) shape[i] = parray.getShape(i); ByteBuffer bb = parray.getPrimdata().asReadOnlyByteBuffer(); bb.order(byteOrder); Array data = Array.factory(dataType, shape, bb); CoverageCoordSys csys = dataResponse.findCoordSys( parray.getCoordSysName()); if (csys == null) throw new IllegalStateException("Misformed response - no coordsys"); return new GeoReferencedArray(parray.getCoverageName(), dataType, data, csys); }
static public NcStreamProto.Attribute.Builder encodeAtt(Attribute att) { NcStreamProto.Attribute.Builder attBuilder = NcStreamProto.Attribute.newBuilder(); attBuilder.setName(att.getShortName()); attBuilder.setDataType(convertDataType(att.getDataType())); attBuilder.setLen(att.getLength()); // values if (att.getLength() > 0) { if (att.isString()) { for (int i = 0; i < att.getLength(); i++) attBuilder.addSdata(att.getStringValue(i)); } else { Array data = att.getValues(); ByteBuffer bb = data.getDataAsByteBuffer(); attBuilder.setData(ByteString.copyFrom(bb.array())); } } return attBuilder; }
static Structure decodeStructure(NetcdfFile ncfile, Group g, Structure parent, NcStreamProto.Structure s) { Structure ncvar = (s.getDataType() == ucar.nc2.stream.NcStreamProto.DataType.SEQUENCE) ? new Sequence(ncfile, g, parent, s.getName()) : new Structure(ncfile, g, parent, s.getName()); ncvar.setDataType(convertDataType(s.getDataType())); List<Dimension> dims = new ArrayList<>(6); for (ucar.nc2.stream.NcStreamProto.Dimension dim : s.getShapeList()) { Dimension ncdim = decodeDim(dim); if (!ncdim.isShared()) dims.add(ncdim); else { Dimension d = g.findDimension(ncdim.getShortName()); if (d == null) throw new IllegalStateException("Can find shared dimension " + dim.getName()); dims.add(d); } } ncvar.setDimensions(dims); for (ucar.nc2.stream.NcStreamProto.Attribute att : s.getAttsList()) ncvar.addAttribute(decodeAtt(att)); for (ucar.nc2.stream.NcStreamProto.Variable vp : s.getVarsList()) ncvar.addMemberVariable(decodeVar(ncfile, g, ncvar, vp)); for (NcStreamProto.Structure sp : s.getStructsList()) ncvar.addMemberVariable(decodeStructure(ncfile, g, ncvar, sp)); return ncvar; }
NcStreamProto.ArrayStructureCol.Builder buildNestedStructureData(List<MemberData> mdataList) { NcStreamProto.ArrayStructureCol.Builder result = NcStreamProto.ArrayStructureCol.newBuilder(); for (MemberData nestedMemberData : mdataList) { NcStreamProto.DataCol.Builder nestedBuilder = NcStreamProto.DataCol.newBuilder(); nestedBuilder.setName(nestedMemberData.member.getName()); nestedBuilder.setDataType(NcStream.convertDataType(nestedMemberData.member.getDataType())); nestedBuilder.setNelems(nestedMemberData.nelems); nestedBuilder.setSection(NcStream.encodeSection(nestedMemberData.section)); if (nestedMemberData.member.isVariableLength()) { nestedMemberData.finishVlens(); nestedBuilder.addAllVlens(nestedMemberData.vlens); nestedMemberData.bb.flip(); nestedBuilder.setPrimdata(ByteString.copyFrom(nestedMemberData.bb)); nestedBuilder.setNelems(nestedMemberData.nelems); nestedBuilder.setIsVlen(true); } else if (nestedMemberData.member.getDataType() == DataType.STRING) { nestedBuilder.addAllStringdata(nestedMemberData.stringList); } else if (nestedMemberData.member.getDataType() == DataType.OPAQUE) { nestedBuilder.addAllOpaquedata(nestedMemberData.opaqueList); } else if (nestedMemberData.member.getDataType() == DataType.STRUCTURE) { nestedBuilder.setStructdata(buildNestedStructureData(nestedMemberData.members)); // recurse } else { nestedMemberData.bb.flip(); nestedBuilder.setPrimdata(ByteString.copyFrom(nestedMemberData.bb)); } result.addMemberData(nestedBuilder); } return result; }
static public Attribute decodeAtt(NcStreamProto.Attribute attp) { // BARF LOOK DataType dtOld = decodeAttributeType(attp.getType()); DataType dtNew = convertDataType(attp.getDataType()); DataType dtUse; if (dtNew != DataType.CHAR) dtUse = dtNew; else if (dtOld != DataType.STRING) dtUse = dtOld; else if (attp.getSdataCount() > 0) dtUse = DataType.STRING; else dtUse = DataType.CHAR; int len = attp.getLen(); if (len == 0) // deal with empty attribute return new Attribute(attp.getName(), dtUse); if (dtUse == DataType.STRING) { int lenp = attp.getSdataCount(); if (lenp != len) System.out.println("HEY lenp != len"); if (lenp == 1) return new Attribute(attp.getName(), attp.getSdata(0)); else { Array data = Array.factory(dtUse, new int[]{lenp}); for (int i = 0; i < lenp; i++) data.setObject(i, attp.getSdata(i)); return new Attribute(attp.getName(), data); } } else { ByteString bs = attp.getData(); ByteBuffer bb = ByteBuffer.wrap(bs.toByteArray()); return new Attribute(attp.getName(), Array.factory(dtUse, (int[]) null, bb)); // if null, then use int[]{bb // .limit()} } }
CdmrFeatureProto.Coverage.Builder encodeGrid(Coverage grid) { CdmrFeatureProto.Coverage.Builder builder = CdmrFeatureProto.Coverage.newBuilder(); builder.setName(grid.getName()); builder.setDataType(NcStream.convertDataType(grid.getDataType())); for (Attribute att : grid.getAttributes()) builder.addAtts(NcStream.encodeAtt(att)); builder.setUnits(grid.getUnitsString()); builder.setDescription(grid.getDescription()); builder.setCoordSys(grid.getCoordSysName()); return builder; }
private void decodeMemberData(StructureMembers members, NcStreamProto.DataCol memberData, Section parentSection) throws IOException { String name = memberData.getName(); DataType dataType = NcStream.convertDataType(memberData.getDataType()); Section section = NcStream.decodeSection(memberData.getSection()); if (!memberData.getIsVlen()) { assert memberData.getNelems() == section.computeSize(); } // the dproto section includes parents, remove them Section msection = section.removeFirst(parentSection); if (memberData.getIsVlen()) msection = msection.appendRange(Range.VLEN); StructureMembers.Member result = members.addMember(name, null, null, dataType, msection.getShape()); Array data = decode(memberData, parentSection); result.setDataArray(data); }
public Array decodeVlenData(NcStreamProto.DataCol dproto) throws IOException { DataType dataType = NcStream.convertDataType(dproto.getDataType()); ByteBuffer bb = dproto.getPrimdata().asReadOnlyByteBuffer(); ByteOrder bo = dproto.getBigend() ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; bb.order(bo); Array alldata = Array.factory(dataType, new int[]{dproto.getNelems()}, bb); // flat array IndexIterator all = alldata.getIndexIterator(); Section section = NcStream.decodeSection(dproto.getSection()); Array[] data = new Array[(int) section.computeSize()]; // divide the primitive data into variable length arrays int count = 0; for (int len : dproto.getVlensList()) { Array primdata = Array.factory(dataType, new int[]{len}); IndexIterator prim = primdata.getIndexIterator(); for (int i=0; i<len; i++) { prim.setObjectNext( all.getObjectNext()); // generic } data[count++] = primdata; } // return Array.makeObjectArray(dataType, data[0].getClass(), section.getShape(), data); return Array.makeVlenArray(section.getShape(), data); }
static NcStreamProto.Data encodeDataProto(Variable var, Section section, NcStreamProto.Compress compressionType, ByteOrder bo, int uncompressedLength) { NcStreamProto.Data.Builder builder = NcStreamProto.Data.newBuilder(); builder.setVarName(var.getFullNameEscaped()); builder.setDataType(convertDataType(var.getDataType())); builder.setSection(encodeSection(section)); builder.setCompress(compressionType); if (compressionType != NcStreamProto.Compress.NONE) { builder.setUncompressedSize(uncompressedLength); } builder.setVdata(var.isVariableLength()); builder.setBigend(bo == ByteOrder.BIG_ENDIAN); builder.setVersion(ncstream_data_version); return builder.build(); }
static NcStreamProto.Variable.Builder encodeVar(Variable var, int sizeToCache) throws IOException { NcStreamProto.Variable.Builder builder = NcStreamProto.Variable.newBuilder(); builder.setName(var.getShortName()); builder.setDataType(convertDataType(var.getDataType())); if (var.getDataType().isEnum()) { EnumTypedef enumType = var.getEnumTypedef(); if (enumType != null) builder.setEnumType(enumType.getShortName()); } for (Dimension dim : var.getDimensions()) { builder.addShape(encodeDim(dim)); } for (Attribute att : var.getAttributes()) { builder.addAtts(encodeAtt(att)); } // put small amounts of data in header "immediate mode" if (var.isCaching() && var.getDataType().isNumeric()) { if (var.isCoordinateVariable() || var.getSize() * var.getElementSize() < sizeToCache) { Array data = var.read(); ByteBuffer bb = data.getDataAsByteBuffer(); builder.setData(ByteString.copyFrom(bb.array())); } } return builder; }
static NcStreamProto.Structure.Builder encodeStructure(Structure s) throws IOException { NcStreamProto.Structure.Builder builder = NcStreamProto.Structure.newBuilder(); builder.setName(s.getShortName()); builder.setDataType(convertDataType(s.getDataType())); for (Dimension dim : s.getDimensions()) builder.addShape(encodeDim(dim)); for (Attribute att : s.getAttributes()) builder.addAtts(encodeAtt(att)); for (Variable v : s.getVariables()) { if (v instanceof Structure) builder.addStructs(NcStream.encodeStructure((Structure) v)); else builder.addVars(NcStream.encodeVar(v, -1)); } return builder; }
public CdmrFeatureProto.GeoReferencedArray.Builder encodeGeoReferencedArray(GeoReferencedArray geoArray, boolean deflate) { CdmrFeatureProto.GeoReferencedArray.Builder builder = CdmrFeatureProto.GeoReferencedArray.newBuilder(); builder.setCoverageName(geoArray.getCoverageName()); builder.setDataType(NcStream.convertDataType(geoArray.getDataType())); builder.setVersion(3); // set to >= 3 for proto3 builder.setBigend(ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); int shape[] = geoArray.getData().getShape(); for (int aShape : shape) builder.addShape(aShape); CoverageCoordSys csys = geoArray.getCoordSysForData(); for (String axisName : csys.getAxisNames()) // geoArray.getAxisNames()) // LOOK could use csys.getAxisNames(), but order may be incorrect, must match shape builder.addAxisName(axisName); builder.setCoordSysName(csys.getName()); if (deflate) { builder.setCompress(NcStreamProto.Compress.DEFLATE); long uncompressedSize = geoArray.getData().getSizeBytes(); builder.setUncompressedSize(uncompressedSize); } else { builder.setPrimdata(NcStreamDataCol.copyArrayToByteString( geoArray.getData())); } return builder; }
CdmrFeatureProto.CoordAxis.Builder encodeCoordAxis(CoverageCoordAxis axis) { CdmrFeatureProto.CoordAxis.Builder builder = CdmrFeatureProto.CoordAxis.newBuilder(); builder.setName(axis.getName()); builder.setDataType(NcStream.convertDataType(axis.getDataType())); builder.setAxisType(convertAxisType(axis.getAxisType())); builder.setNvalues(axis.getNcoords());
static public PointStreamProto.PointFeatureCollection encodePointFeatureCollection( String name, String timeUnitString, String altUnits, PointFeature pf) throws IOException { PointStreamProto.PointFeatureCollection.Builder builder = PointStreamProto.PointFeatureCollection.newBuilder(); if (name == null) System.out.printf("HEY null pointstream name%n"); builder.setName(name); builder.setTimeUnit(timeUnitString); if (altUnits != null) { builder.setAltUnit(altUnits); } StructureData sdata = pf.getDataAll(); StructureMembers sm = sdata.getStructureMembers(); for (StructureMembers.Member m : sm.getMembers()) { PointStreamProto.Member.Builder mbuilder = PointStreamProto.Member.newBuilder(); mbuilder.setName(m.getName()); if (null != m.getDescription()) mbuilder.setDesc(m.getDescription()); if (null != m.getUnitsString()) mbuilder.setUnits(m.getUnitsString()); mbuilder.setDataType(NcStream.convertDataType(m.getDataType())); mbuilder.setSection(NcStream.encodeSection(new ucar.ma2.Section(m.getShape()))); builder.addMembers(mbuilder); } return builder.build(); }
DataType dataType = NcStream.convertDataType(dproto.getDataType()); Section section = (dataType == DataType.SEQUENCE) ? new Section() : NcStream.decodeSection(dproto.getSection()); if (!dproto.getIsVlen()) {