/** * Set the data type * * @param dataType set to this value */ public void setDataType(DataType dataType) { if (immutable) throw new IllegalStateException("Cant modify"); this.dataType = dataType; this.elementSize = getDataType().getSize(); /* why is this needed ?? EnumTypedef etd = getEnumTypedef(); if (etd != null) { DataType etdtype = etd.getBaseType(); if (dataType != etdtype) log.error("Variable.setDataType: enum basetype mismatch: {} != {}", etdtype, dataType); /* DataType basetype = null; if (dataType == DataType.ENUM1) basetype = DataType.BYTE; else if (dataType == DataType.ENUM2) basetype = DataType.SHORT; else if (dataType == DataType.ENUM4) basetype = DataType.INT; else basetype = etdtype; if (etdtype != null && dataType != etdtype) else etd.setBaseType(basetype); } */ }
/** * Set the data type * * @param dataType set to this value */ public void setDataType(DataType dataType) { if (immutable) throw new IllegalStateException("Cant modify"); this.dataType = dataType; this.elementSize = getDataType().getSize(); /* why is this needed ?? EnumTypedef etd = getEnumTypedef(); if (etd != null) { DataType etdtype = etd.getBaseType(); if (dataType != etdtype) log.error("Variable.setDataType: enum basetype mismatch: {} != {}", etdtype, dataType); /* DataType basetype = null; if (dataType == DataType.ENUM1) basetype = DataType.BYTE; else if (dataType == DataType.ENUM2) basetype = DataType.SHORT; else if (dataType == DataType.ENUM4) basetype = DataType.INT; else basetype = etdtype; if (etdtype != null && dataType != etdtype) else etd.setBaseType(basetype); } */ }
public static ByteString copyArrayToByteString(Array data) { int nbytes = (int) data.getSizeBytes(); if (nbytes < 0) { System.out.printf("copyArrayToByteString neg byte size %d dataType = %d data size %d shape = %s%n", nbytes, data.getDataType().getSize(), data.getSize(), Misc.showInts(data.getShape())); } ByteBuffer bb = ByteBuffer.allocate(nbytes); bb.order(ByteOrder.nativeOrder()); copyArrayToBB(data, false, bb); bb.flip(); return ByteString.copyFrom(bb); }
/** * Get the total number of bytes in the array. * * @return total number of bytes in the array */ public long getSizeBytes() { DataType dtype = DataType.getType(this); return indexCalc.getSize() * dtype.getSize(); }
/** * Get the total number of bytes in the array. * * @return total number of bytes in the array */ public long getSizeBytes() { DataType dtype = DataType.getType(getElementType()); return indexCalc.getSize() * dtype.getSize(); }
/** * Get the total number of bytes in the array. * * @return total number of bytes in the array */ public long getSizeBytes() { DataType dtype = DataType.getType( getElementType()); return indexCalc.getSize() * dtype.getSize(); }
/** * Get the total size in bytes. This does not have to match the VariableSimpleIF. * * Note that this will not be correct when containing a member of type Sequence, or String, since those * are variable length. In that case * * @return total size in bytes */ public int getSizeBytes() { if (getDataType() == DataType.SEQUENCE) return getDataType().getSize(); else if (getDataType() == DataType.STRING) return getDataType().getSize(); else if (getDataType() == DataType.STRUCTURE) return size * members.getStructureSize(); else return size * getDataType().getSize(); }
/** * Get the total size in bytes. This does not have to match the VariableSimpleIF. * * Note that this will not be correct when containing a member of type Sequence, or String, since those * are variable length. In that case * * @return total size in bytes */ public int getSizeBytes() { if (getDataType() == DataType.SEQUENCE) return getDataType().getSize(); else if (getDataType() == DataType.STRING) return getDataType().getSize(); else if (getDataType() == DataType.STRUCTURE) return size * members.getStructureSize(); //else if (this.isVariableLength()) // return 0; // do not know else return size * getDataType().getSize(); }
/** * Get the total size in bytes. This does not have to match the VariableSimpleIF. * * Note that this will not be correct when containing a member of type Sequence, or String, since those * are variable length. In that case * * @return total size in bytes */ public int getSizeBytes() { if (getDataType() == DataType.SEQUENCE) return getDataType().getSize(); else if (getDataType() == DataType.STRING) return getDataType().getSize(); else if (getDataType() == DataType.STRUCTURE) return size * members.getStructureSize(); else if(this.isVariableLength()) return 0; // do not know else return size * getDataType().getSize(); }
public long getSizeInBytes() { Section section = new Section(coordSys.getShape()); return section.computeSize() * getDataType().getSize(); }
private void createEnumType(Group4 g4, EnumTypedef en) throws IOException { IntByReference typeidp = new IntByReference(); String name = en.getShortName(); DataType enumbase = en.getBaseType(); int basetype = NC_NAT; if(enumbase == DataType.ENUM1) basetype = Nc4prototypes.NC_BYTE; else if(enumbase == DataType.ENUM2) basetype = Nc4prototypes.NC_SHORT; else if(enumbase == DataType.ENUM4) basetype = Nc4prototypes.NC_INT; int ret = nc4.nc_def_enum(g4.grpid, basetype, name, typeidp); if(ret != 0) throw new IOException(nc4.nc_strerror(ret) + " on\n" + en); int typeid = typeidp.getValue(); if(DEBUG) System.out.printf("added enum type %s (typeid %d)%n", name, typeid); Map<Integer, String> emap = en.getMap(); for(Map.Entry<Integer, String> entry : emap.entrySet()) { IntByReference val = new IntByReference(entry.getKey()); ret = nc4.nc_insert_enum(g4.grpid, typeid, (String) entry.getValue(), val); if(ret != 0) throw new IOException(nc4.nc_strerror(ret) + " on\n" + entry.getValue()); if(DEBUG) System.out.printf(" added enum type member %s: %d%n", entry.getValue(), entry.getKey()); } // keep track of the User Defined types UserType ut = new UserType( g4.grpid, typeid, name, en.getBaseType().getSize(), basetype, (long) emap.size(), NC_ENUM); userTypes.put(typeid, ut); en.annotate(UserType.class,ut); // dont know the varid yet }
/** * Set the data type * * @param dataType set to this value */ public void setDataType(DataType dataType) { if (immutable) throw new IllegalStateException("Cant modify"); if(getShortName().equals("IR") && dataType == DataType.SHORT) { DataType old = getDataType(); } this.dataType = dataType; this.elementSize = getDataType().getSize(); EnumTypedef etd = getEnumTypedef(); if(etd != null) { DataType etdtype = etd.getBaseType(); DataType basetype = null; if(dataType == DataType.ENUM1) basetype = DataType.BYTE; else if(dataType == DataType.ENUM2) basetype = DataType.SHORT; else if(dataType == DataType.ENUM1) basetype = DataType.INT; else basetype = etdtype; if(etdtype != null && basetype != etdtype) log.error("Variable.setDataType: enum basetype mismatch: "+basetype.name()); else etd.setBaseType(basetype); } }
private Vinfo writeVar(DataOutputStream stream, Variable var, int offset) throws IOException { int hsize = 0; hsize += writeString(stream, N3iosp.makeValidNetcdfObjectName( var.getShortName())); // dimensions int vsize = var.getDataType().getSize(); List<Dimension> dims = var.getDimensions(); if (null != stream) stream.writeInt(dims.size()); hsize += 4; for (Dimension dim : dims) { int dimIndex = findDimensionIndex(dim); if (null != stream) stream.writeInt(dimIndex); hsize += 4; if (!dim.isUnlimited()) vsize *= dim.getLength(); } int pad = (usePadding) ? N3header.padding(vsize) : 0; vsize += pad; // variable attributes hsize += writeAtts(stream, var.getAttributes()); // data type, variable size, beginning file position int type = N3header.getType(var.getDataType()); if (null != stream) { stream.writeInt(type); stream.writeInt(vsize); stream.writeInt(offset); } hsize += 12; //if (debug) out.println(" name= "+name+" type="+type+" vsize="+vsize+" begin= "+begin+" isRecord="+isRecord+"\n"); return new Vinfo(var, hsize, vsize, offset, pad, var.isUnlimited()); }
private Vinfo writeVar(DataOutputStream stream, Variable var, int offset) throws IOException { int hsize = 0; hsize += writeString(stream, N3iosp.makeValidNetcdfObjectName( var.getShortName())); // dimensions int vsize = var.getDataType().getSize(); List<Dimension> dims = var.getDimensions(); if (null != stream) stream.writeInt(dims.size()); hsize += 4; for (Dimension dim : dims) { int dimIndex = findDimensionIndex(dim); if (null != stream) stream.writeInt(dimIndex); hsize += 4; if (!dim.isUnlimited()) vsize *= dim.getLength(); } int pad = (usePadding) ? N3header.padding(vsize) : 0; vsize += pad; // variable attributes hsize += writeAtts(stream, var.getAttributes()); // data type, variable size, beginning file position int type = N3header.getType(var.getDataType()); if (null != stream) { stream.writeInt(type); stream.writeInt(vsize); stream.writeInt(offset); } hsize += 12; //if (debug) out.println(" name= "+name+" type="+type+" vsize="+vsize+" begin= "+begin+" isRecord="+isRecord+"\n"); return new Vinfo(var, hsize, vsize, offset, pad, var.isUnlimited()); }
private Vinfo writeVar(DataOutputStream stream, Variable var, int offset) throws IOException { int hsize = 0; hsize += writeString(stream, N3iosp.makeValidNetcdfObjectName( var.getShortName())); // dimensions int vsize = var.getDataType().getSize(); List<Dimension> dims = var.getDimensions(); if (null != stream) stream.writeInt(dims.size()); hsize += 4; for (Dimension dim : dims) { int dimIndex = findDimensionIndex(dim); if (null != stream) stream.writeInt(dimIndex); hsize += 4; if (!dim.isUnlimited()) vsize *= dim.getLength(); } int pad = (usePadding) ? N3header.padding(vsize) : 0; vsize += pad; // variable attributes hsize += writeAtts(stream, var.getAttributes()); // data type, variable size, beginning file position int type = N3header.getType(var.getDataType()); if (null != stream) { stream.writeInt(type); stream.writeInt(vsize); stream.writeInt(offset); } hsize += 12; //if (debug) out.println(" name= "+name+" type="+type+" vsize="+vsize+" begin= "+begin+" isRecord="+isRecord+"\n"); return new Vinfo(var, hsize, vsize, offset, pad, var.isUnlimited()); }
private Attribute readEnumAttValues(int grpid, int varid, String attname, int len, UserType userType) throws IOException { int ret; DataType dtype = convertDataType(userType.baseTypeid).dt; int elemSize = dtype.getSize(); ByteBuffer bb = ByteBuffer.allocate(len * elemSize); ret = nc4.nc_get_att(grpid, varid, attname, bb); if (ret != 0) throw new IOException(ret + ": " + nc4.nc_strerror(ret)); Array data = convertByteBuffer(bb, userType.baseTypeid, new int[]{len}); IndexIterator ii = data.getIndexIterator(); if (len == 1) { String val = userType.e.lookupEnumString(ii.getIntNext()); return new Attribute(attname, val); } else { ArrayObject.D1 attArray = (ArrayObject.D1) Array.factory(DataType.STRING, new int[]{len}); for (int i = 0; i < len; i++) { int val = ii.getIntNext(); String vals = userType.e.lookupEnumString(val); if (vals == null) throw new IOException("Illegal enum val " + val + " for attribute " + attname); attArray.set(i, vals); } return new Attribute(attname, attArray); } }
void encodeVlenData(NcStreamProto.DataRow.Builder builder, Section section, Array data) { if (!(data instanceof ArrayObject)) throw new IllegalStateException("Unknown class for OPAQUE =" + data.getClass().getName()); IndexIterator iter = data.getIndexIterator(); int count = 0; while (iter.hasNext()) { Array varray = (Array) iter.next(); int vlensize = (int) varray.getSize(); builder.addVlens(vlensize); count += vlensize; } builder.setNelems(count); Section ssection = section.removeLast(); builder.setSection(NcStream.encodeSection(ssection)); assert section.computeSize() == count; int nbytes = count * data.getDataType().getSize(); ByteBuffer bb = ByteBuffer.allocate(nbytes); // copyArrayToBB(data, bb); iter = data.getIndexIterator(); while (iter.hasNext()) { Array varray = (Array) iter.next(); copyArrayToBB(varray, bb); } }
private Attribute readEnumAttValues(int grpid, int varid, String attname, int len, UserType userType) throws IOException { int ret; DataType dtype = convertDataType(userType.baseTypeid).dt; int elemSize = dtype.getSize(); byte[] bbuff = new byte[len * elemSize]; ret = nc4.nc_get_att(grpid, varid, attname, bbuff); if (ret != 0) throw new IOException(ret + ": " + nc4.nc_strerror(ret)); ByteBuffer bb = ByteBuffer.wrap(bbuff); Array data = convertByteBuffer(bb, userType.baseTypeid, new int[]{len}); IndexIterator ii = data.getIndexIterator(); if (len == 1) { String val = userType.e.lookupEnumString(ii.getIntNext()); return new Attribute(attname, val); } else { ArrayObject.D1 attArray = (ArrayObject.D1) Array.factory(DataType.STRING, new int[]{len}); for (int i = 0; i < len; i++) { int val = ii.getIntNext(); String vals = userType.e.lookupEnumString(val); if (vals == null) throw new IOException("Illegal enum val " + val + " for attribute " + attname); attArray.set(i, vals); } return new Attribute(attname, attArray); } }
assert ssection.computeSize() == count; int nbytes = nelems * data.getDataType().getSize(); ByteBuffer bb = ByteBuffer.allocate(nbytes); bb.order(ByteOrder.nativeOrder());
@Test public void testRequestNetCDFCompression() throws Exception { boolean isNC4Available = NetCDFUtilities.isNC4CAvailable(); if (!isNC4Available && LOGGER.isLoggable(Level.INFO)) { LOGGER.info("NetCDF C library not found. NetCDF4 output will not be created"); } // http response from the request inside the string MockHttpServletResponse response = getAsServletResponse( "ows?request=GetCoverage&service=WCS&version=2.0.1" + "&coverageId=wcs__visibilityCompressed&format=application/x-netcdf4"); assertNotNull(response); assertEquals( (isNC4Available ? "application/x-netcdf4" : "application/xml"), response.getContentType()); if (isNC4Available) { byte[] netcdfOut = getBinary(response); File file = File.createTempFile("netcdf", "outCompressed.nc", new File("./target")); FileUtils.writeByteArrayToFile(file, netcdfOut); NetcdfDataset dataset = NetcdfDataset.openDataset(file.getAbsolutePath()); assertNotNull(dataset); Variable var = dataset.findVariable(STANDARD_NAME); assertNotNull(var); final long varByteSize = var.getSize() * var.getDataType().getSize(); // The output file is smaller than the size of the underlying variable. // Compression successfully occurred assertTrue(netcdfOut.length < varByteSize); dataset.close(); } }