/** * Constructor * * @param want skip any nodes that are before this section * @throws IOException on error */ DataChunkIterator(Section want) throws IOException { root = new Node(rootNodeAddress, -1); // should we cache the nodes ??? wantOrigin = (want != null) ? want.getOrigin() : null; root.first(wantOrigin); }
/** * Constructor * * @param want skip any nodes that are before this section * @throws IOException on error */ DataChunkIterator(Section want) throws IOException { root = new Node(rootNodeAddress, -1); // should we cache the nodes ??? wantOrigin = (want != null) ? want.getOrigin() : null; root.first(wantOrigin); }
/** * Constructor * * @param want skip any nodes that are before this section * @param nChunkDim number of chunk dimensions - may be less than the offset[] length * @throws IOException on error */ DataChunkIteratorNoFilter(Section want, int nChunkDim) throws IOException { this.nChunkDim = nChunkDim; root = new Node(rootNodeAddress, -1); // should we cache the nodes ??? int[] wantOrigin = (want != null) ? want.getOrigin() : null; root.first(wantOrigin); }
/** * Constructor * * @param want skip any nodes that are before this section * @throws IOException on error */ DataChunkIterator(Section want) throws IOException { root = new Node(rootNodeAddress, -1); // should we cache the nodes ??? wantOrigin = (want != null) ? want.getOrigin() : null; root.first(wantOrigin); }
/** * Constructor * * @param want skip any nodes that are before this section * @param nChunkDim number of chunk dimensions - may be less than the offset[] length * @throws IOException on error */ DataChunkIteratorNoFilter(Section want, int nChunkDim) throws IOException { this.nChunkDim = nChunkDim; root = new Node(rootNodeAddress, -1); // should we cache the nodes ??? int[] wantOrigin = (want != null) ? want.getOrigin() : null; root.first(wantOrigin); }
/** * Constructor * * @param want skip any nodes that are before this section * @param nChunkDim number of chunk dimensions - may be less than the offset[] length * @throws IOException on error */ DataChunkIteratorNoFilter(Section want, int nChunkDim) throws IOException { this.nChunkDim = nChunkDim; root = new Node(rootNodeAddress, -1); // should we cache the nodes ??? int[] wantOrigin = (want != null) ? want.getOrigin() : null; root.first(wantOrigin); }
private void writeCompoundData(Structure s, UserType userType, int grpid, int varid, int typeid, Section section, ArrayStructure values) throws IOException, InvalidRangeException { SizeT[] origin = convertSizeT(section.getOrigin()); SizeT[] shape = convertSizeT(section.getShape()); SizeT[] stride = convertSizeT(section.getStride()); ArrayStructureBB valuesBB = StructureDataDeep.copyToArrayBB(s, values, ByteOrder.nativeOrder()); // LOOK embedded strings getting lost ?? ByteBuffer bbuff = valuesBB.getByteBuffer(); if (debugCompound) System.out.printf("writeCompoundData variable %s (grpid %d varid %d) %n", s.getShortName(), grpid, varid); // write the data // int ret = nc4.nc_put_var(grpid, varid, bbuff); int ret; if(section.isStrided()) ret = nc4.nc_put_vars(grpid, varid, origin, shape, stride, bbuff.array()); else ret = nc4.nc_put_vara(grpid, varid, origin, shape, bbuff.array()); if (ret != 0) throw new IOException(errMessage("nc_put_vars", ret, grpid, varid)); }
TileLayout(Section localSection, Section wantSection) throws InvalidRangeException { Section dataSection = localSection.compact(); Section resultSection = wantSection.compact(); if (debug) System.out.println(" resultSection: " + resultSection); if (debug) System.out.println(" dataSection: " + dataSection); int rank = dataSection.getRank(); // total elements to transfer total = (int) dataSection.computeSize(); // figure out the offset long product = 1; startElem = 0; // offset in want for (int ii = rank - 1; ii >= 0; ii--) { int d = dataSection.getOrigin(ii) - resultSection.getOrigin(ii); if (d > 0) startElem += product * d; product *= resultSection.getShape(ii); } resultPos = startElem; // we will use an Index object to keep track of the chunks // last range length is nelems; reduce index rank nelems = localSection.getShape(rank - 1); int[] stride = new int[rank - 1]; int[] shape = new int[rank - 1]; product = resultSection.getShape(rank - 1); for (int ii = rank - 2; ii >= 0; ii--) { stride[ii] = (int) product; shape[ii] = dataSection.getShape(ii); product *= resultSection.getShape(ii); } index = new Index(shape, stride); }
private void writeCompoundData(Structure s, UserType userType, int grpid, int varid, int typeid, Section section, ArrayStructure values) throws IOException, InvalidRangeException { SizeT[] origin = convertSizeT(section.getOrigin()); SizeT[] shape = convertSizeT(section.getShape()); SizeT[] stride = convertSizeT(section.getStride()); ArrayStructureBB valuesBB = StructureDataDeep.copyToArrayBB(s, values, ByteOrder.nativeOrder()); // LOOK embedded strings getting lost ?? ByteBuffer bbuff = valuesBB.getByteBuffer(); if (debugCompound) System.out.printf("writeCompoundData variable %s (grpid %d varid %d) %n", s.getShortName(), grpid, varid); //IntByReference val = new IntByReference(); //int ret2 = nc4.nc_inq_var_fletcher32(grpid, varid, val); //System.out.printf("%d = %d%n", ret2, val.getValue()); // write the data // int ret = nc4.nc_put_var(grpid, varid, bbuff); int ret; if (section.isStrided()) ret = nc4.nc_put_vars(grpid, varid, origin, shape, stride, bbuff.array()); else ret = nc4.nc_put_vara(grpid, varid, origin, shape, bbuff.array()); if (ret != 0) throw new IOException(errMessage("nc_put_vars", ret, grpid, varid)); }
TileLayout(Section localSection, Section wantSection) throws InvalidRangeException { Section dataSection = localSection.compact(); Section resultSection = wantSection.compact(); if (debug) System.out.println(" resultSection: " + resultSection); if (debug) System.out.println(" dataSection: " + dataSection); int rank = dataSection.getRank(); // total elements to transfer total = (int) dataSection.computeSize(); // figure out the offset long product = 1; startElem = 0; // offset in want for (int ii = rank - 1; ii >= 0; ii--) { int d = dataSection.getOrigin(ii) - resultSection.getOrigin(ii); if (d > 0) startElem += product * d; product *= resultSection.getShape(ii); } resultPos = startElem; // we will use an Index object to keep track of the chunks // last range length is nelems; reduce index rank nelems = localSection.getShape(rank - 1); int[] stride = new int[rank - 1]; int[] shape = new int[rank - 1]; product = resultSection.getShape(rank - 1); for (int ii = rank - 2; ii >= 0; ii--) { stride[ii] = (int) product; shape[ii] = dataSection.getShape(ii); product *= resultSection.getShape(ii); } index = new Index(shape, stride); }
private Array readCompound(int grpid, int varid, Section section, UserType userType) throws IOException { SizeT[] origin = convertSizeT(section.getOrigin()); SizeT[] shape = convertSizeT(section.getShape()); SizeT[] stride = convertSizeT(section.getStride()); int len = (int) section.computeSize(); int buffSize = len * userType.size; byte[] bbuff = new byte[buffSize]; // read in the data int ret; ret = nc4.nc_get_vars(grpid, varid, origin, shape, stride, bbuff); if (ret != 0) throw new IOException(ret + ": " + nc4.nc_strerror(ret)); ByteBuffer bb = ByteBuffer.wrap(bbuff); bb.order(ByteOrder.nativeOrder()); // c library returns in native order i hope /* This does not seem right since the user type does not normally appear in the CDM representation. */ StructureMembers sm = createStructureMembers(userType); ArrayStructureBB asbb = new ArrayStructureBB(sm, section.getShape(), bb, 0); // find and convert String and vlen fields, put on asbb heap int destPos = 0; for (int i = 0; i < len; i++) { // loop over each structure convertHeap(asbb, destPos, sm); destPos += userType.size; } return asbb; }
private Array readCompound(int grpid, int varid, Section section, UserType userType) throws IOException { long[] origin = convert(section.getOrigin()); long[] shape = convert(section.getShape()); long[] stride = convert(section.getStride()); int len = (int) section.computeSize(); int buffSize = len * userType.size; ByteBuffer bbuff = ByteBuffer.allocate(buffSize); bbuff.order(ByteOrder.nativeOrder()); // c library returns in native order i hope // read in the data int ret; if (isStride1(section.getStride())) ret = nc4.nc_get_vara(grpid, varid, origin, shape, bbuff); else ret = nc4.nc_get_vars(grpid, varid, origin, shape, stride, bbuff); if (ret != 0) throw new IOException(ret + ": " + nc4.nc_strerror(ret)); StructureMembers sm = createStructureMembers(userType); ArrayStructureBB asbb = new ArrayStructureBB(sm, section.getShape(), bbuff, 0); // find and convert Strings and vlens, put on asbb heap int destPos = 0; for (int i = 0; i < len; i++) { // loop over each structure convertHeap(asbb, destPos, sm); destPos += userType.size; } return asbb; }
private Array readOpaque(int grpid, int varid, Section section, int size) throws IOException, InvalidRangeException { int ret; long[] origin = convert(section.getOrigin()); long[] shape = convert(section.getShape()); long[] stride = convert(section.getStride()); int len = (int) section.computeSize(); ByteBuffer bb = ByteBuffer.allocate(len * size); if (isStride1(section.getStride())) ret = nc4.nc_get_vara(grpid, varid, origin, shape, bb); else ret = nc4.nc_get_vars(grpid, varid, origin, shape, stride, bb); if (ret != 0) throw new IOException(ret + ": " + nc4.nc_strerror(ret)); byte[] entire = bb.array(); // fix: this is ignoring the rank of section. // was: ArrayObject values = new ArrayObject(ByteBuffer.class, new int[]{len}); int[] intshape = new int[shape.length]; for (int i = 0; i < intshape.length; i++) { intshape[i] = (int) shape[i]; } ArrayObject values = new ArrayObject(ByteBuffer.class, intshape); int count = 0; IndexIterator ii = values.getIndexIterator(); while (ii.hasNext()) { ii.setObjectNext(ByteBuffer.wrap(entire, count * size, size)); count++; } return values; }
private int writeEnumData(Variable v, UserType userType, int grpid, int varid, int typeid, Section section, Array values) throws IOException, InvalidRangeException { int ret = 0; SizeT[] origin = convertSizeT(section.getOrigin()); SizeT[] shape = convertSizeT(section.getShape()); boolean isUnsigned = isUnsigned(typeid); int sectionLen = (int) section.computeSize(); assert values.getSize() == sectionLen; int[] secStride = section.getStride(); boolean stride1 = isStride1(secStride); ByteBuffer bb = values.getDataAsByteBuffer(ByteOrder.nativeOrder()); byte[] data = bb.array(); if(stride1) { ret = nc4.nc_put_vara(grpid, varid, origin, shape, data); } else { SizeT[] stride = convertSizeT(secStride); ret = nc4.nc_put_vars(grpid, varid, origin, shape, stride, data); } return ret; }
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { // subset // Range[] section = Range.toArray( sectionList); int[] origin = section.getOrigin(); int[] shape = section.getShape(); int[] stride = section.getStride(); FysatHeader.Vinfo vinfo = (FysatHeader.Vinfo) v2.getSPobject(); if (headerParser.getCompressType() == 0) return readData(v2, vinfo.begin, origin, shape, stride); else if (headerParser.getCompressType() == 2) return readCompressedData(v2, vinfo.begin, origin, shape, stride); else if (headerParser.getCompressType() == 1) return readCompressedZlib(v2, vinfo.begin, vinfo.nx, vinfo.ny, origin, shape, stride); else return null; }
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { // subset // Range[] section = Range.toArray( sectionList); int[] origin = section.getOrigin(); int[] shape = section.getShape(); int[] stride = section.getStride(); FysatHeader.Vinfo vinfo = (FysatHeader.Vinfo) v2.getSPobject(); if (headerParser.getCompressType() == 0) return readData(v2, vinfo.begin, origin, shape, stride); else if (headerParser.getCompressType() == 2) return readCompressedData(v2, vinfo.begin, origin, shape, stride); else if (headerParser.getCompressType() == 1) return readCompressedZlib(v2, vinfo.begin, vinfo.nx, vinfo.ny, origin, shape, stride); else return null; }
private Array readOpaque(int grpid, int varid, Section section, int size) throws IOException, InvalidRangeException { int ret; SizeT[] origin = convertSizeT(section.getOrigin()); SizeT[] shape = convertSizeT(section.getShape()); SizeT[] stride = convertSizeT(section.getStride());
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { // subset int[] origin = section.getOrigin(); int[] shape = section.getShape(); int[] stride = section.getStride(); Giniheader.Vinfo vinfo = (Giniheader.Vinfo) v2.getSPobject(); int[] levels = vinfo.levels; if (headerParser.gini_GetCompressType() == 0) return readData(v2, vinfo.begin, origin, shape, stride, levels); else if (headerParser.gini_GetCompressType() == 2) return readCompressedData(v2, vinfo.begin, origin, shape, stride, levels); else if (headerParser.gini_GetCompressType() == 1) return readCompressedZlib(v2, vinfo.begin, vinfo.nx, vinfo.ny, origin, shape, stride, levels); else return null; }
private Array readOpaque(int grpid, int varid, Section section, int size) throws IOException, InvalidRangeException { int ret; SizeT[] origin = convertSizeT(section.getOrigin()); SizeT[] shape = convertSizeT(section.getShape()); SizeT[] stride = convertSizeT(section.getStride());
public void testReadDataSection(Variable v, Section s) throws InvalidRangeException, IOException { logger.debug("Read Section {}", s); Array data = v.read(s); assert data.getRank() == 2; assert data.getSize() == s.computeSize(); assert data.getShape()[0] == s.getShape(0); assert data.getShape()[1] == s.getShape(1); assert data.getElementType() == double.class; int startRow = s.getOrigin(0); int startCol = s.getOrigin(1); int strideRow = s.getStride(0); int strideCol = s.getStride(1); int[] shape = data.getShape(); Index tIndex = data.getIndex(); for (int row = 0; row < shape[0]; row++) for (int col = 0; col < shape[1]; col++) { double val = data.getDouble( tIndex.set(row, col)); double truth = getVal(startRow + row*strideRow, startCol + col*strideCol); Assert2.assertNearlyEquals(val, truth); } } }