private ArrayStructure convertStructure(Structure s, Layout layout, int[] shape, byte[] byteArray) throws IOException, InvalidRangeException { // create StructureMembers - must set offsets StructureMembers sm = s.makeStructureMembers(); int calcSize = ArrayStructureBB.setOffsets(sm); // standard //ArrayStructureBB.showOffsets(sm, new Indent(2), new Formatter(System.out)); // special offset setting boolean hasHeap = convertStructure(s, sm); //ArrayStructureBB.showOffsets(sm, new Indent(2), new Formatter(System.out)); int recSize = layout.getElemSize(); if (recSize < calcSize) { log.error("calcSize = %d actualSize = %d%n", calcSize, recSize); throw new IOException("H5iosp illegal structure size " + s.getFullName()); } sm.setStructureSize(recSize); // place data into an ArrayStructureBB ByteBuffer bb = ByteBuffer.wrap(byteArray); ArrayStructureBB asbb = new ArrayStructureBB(sm, shape, bb, 0); // strings are stored on the heap, and must be read separately if (hasHeap) { int destPos = 0; for (int i = 0; i < layout.getTotalNelems(); i++) { // loop over each structure convertHeap(asbb, destPos, sm); destPos += layout.getElemSize(); } } return asbb; }
public void showCompress(Formatter f) throws IOException { NetcdfFile ncfile = iosp.getNetcdfFile(); Size totalSize = new Size(0,0); for (Variable v : ncfile.getVariables()) { H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject(); showCompress(v, vinfo, totalSize, f); } f.format("%n"); f.format(" total bytes = %d%n", totalSize.nominal); f.format(" total storage = %d%n", totalSize.storage); f.format(" compression = %f%n", totalSize.getRatio()); f.format(" nchunks = %d%n", totalSize.count); File raf = new File(ncfile.getLocation()); f.format(" file size = %d%n", raf.length()); f.format(" overhead = %f%n", ((float) raf.length()/totalSize.storage)); }
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { H5header.Vinfo vinfo = (H5header.Vinfo) v2.getSPobject(); return readData(v2, vinfo.dataPos, section); }
long address = chunk.getSrcPos() + layout.getElemSize() * i; Array vlenArray = headerParser.getHeapDataArray(address, readType, endian); data[count++] = (typeInfo.base.hdfType == 7) ? convertReference(vlenArray) : vlenArray; return convertStructure((Structure) v, layout, shape, byteArray); // LOOK return readDataPrimitive(layout, dataType, shape, fillValue, endian, true);
int destPos = 0; for (int i = 0; i < layout.getTotalNelems(); i++) { // loop over each structure h5iosp.convertHeap(asbb, destPos, sm); destPos += layout.getElemSize(); Array vlenArray = getHeapDataArray(address, readType, endian); if (vinfo.typeInfo.base.hdfType == 7) data[count++] = h5iosp.convertReference(vlenArray); else data[count++] = vlenArray; Object data = h5iosp.readDataPrimitive(layout, dataType, shape, null, endian, false); Array dataArray;
layout = new H5tiledLayoutBB(v2, wantSection, raf, vinfo.mfp.getFilters(), bo); if (vinfo.typeInfo.isVString) { data = readFilteredStringData((LayoutBB) layout); } else{ data = IospHelper.readDataFill((LayoutBB) layout, v2.getDataType(), vinfo.getFillValue()); layout = new LayoutRegular(dataPos, elemSize, v2.getShape(), wantSection); data = readData(vinfo, v2, layout, readDtype, wantSection.getShape(), fillValue, endian); return (Array) data; else if (dataType == DataType.STRUCTURE) return convertStructure((Structure) v2, layout, wantSection.getShape(), (byte[]) data); // LOOK else return Array.factory(dataType, wantSection.getShape(), data);
layout = new LayoutRegular(dataPos, elemSize, v2.getShape(), wantSection); data = readData(vinfo, v2, layout, readDtype, wantSection.getShape(), fillValue, endian); return (Array) data; else if (dataType == DataType.STRUCTURE) return convertStructure((Structure) v2, layout, wantSection.getShape(), (byte[]) data); // LOOK else return Array.factory(dataType.getPrimitiveClassType(), wantSection.getShape(), data);
private boolean convertStructure(Structure s, StructureMembers sm ) { boolean hasHeap = false; for (StructureMembers.Member m : sm.getMembers()) { Variable v2 = s.findVariable(m.getName()); assert v2 != null; H5header.Vinfo vm = (H5header.Vinfo) v2.getSPobject(); // apparently each member may have seperate byte order (!!!??) if (vm.typeInfo.endian >= 0) m.setDataObject(vm.typeInfo.endian == RandomAccessFile.LITTLE_ENDIAN ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); // vm.dataPos : offset since start of Structure m.setDataParam((int) vm.dataPos); // track if there is a heap if (v2.getDataType() == DataType.STRING || v2.isVariableLength()) hasHeap = true; // recurse if (v2 instanceof Structure) { Structure nested = (Structure) v2; StructureMembers nestSm = nested.makeStructureMembers(); m.setStructureMembers(nestSm); hasHeap |= convertStructure(nested, nestSm); } } return hasHeap; }
public L1BGociFileReader(L1BGociProductReaderPlugIn readerPlugIn) { super(readerPlugIn); H5iosp.setDebugFlags(new DebugFlagsImpl("HdfEos/turnOff")); }
private static IOServiceProvider getIOSpi(byte[] buffer) { IOServiceProvider spi = null; if (testMagicBytes(buffer, NC3_MAGIC)) { spi = new N3raf(); } else if (testMagicBytes(buffer, NC3_MAGIC_LONG)) { spi = new N3raf(); } else if (testMagicBytes(buffer, H4_MAGIC)) { spi = new H4iosp(); } else if (testMagicBytes(buffer, H5_MAGIC)) { spi = new H5iosp(); } return spi; }
long address = chunk.getSrcPos() + layout.getElemSize() * i; Array vlenArray = headerParser.getHeapDataArray(address, readType, endian); data[count++] = (typeInfo.base.hdfType == 7) ? convertReference(vlenArray) : vlenArray; return convertStructure((Structure) v, layout, shape, byteArray); // LOOK return readDataPrimitive(layout, dataType, shape, fillValue, endian, true);
int destPos = 0; for (int i = 0; i < layout.getTotalNelems(); i++) { // loop over each structure h5iosp.convertHeap(asbb, destPos, sm); destPos += layout.getElemSize(); Array vlenArray = getHeapDataArray(address, readType, endian); if (vinfo.typeInfo.base.hdfType == 7) data[count++] = h5iosp.convertReference(vlenArray); else data[count++] = vlenArray; Object data = h5iosp.readDataPrimitive(layout, dataType, shape, null, endian, false); Array dataArray;
layout = new LayoutRegular(dataPos, elemSize, v2.getShape(), wantSection); data = readData(vinfo, v2, layout, readDtype, wantSection.getShape(), fillValue, endian); return (Array) data; else if (dataType == DataType.STRUCTURE) return convertStructure((Structure) v2, layout, wantSection.getShape(), (byte[]) data); // LOOK else return Array.factory(dataType.getPrimitiveClassType(), wantSection.getShape(), data);
private boolean convertStructure(Structure s, StructureMembers sm ) { boolean hasHeap = false; for (StructureMembers.Member m : sm.getMembers()) { Variable v2 = s.findVariable(m.getName()); assert v2 != null; H5header.Vinfo vm = (H5header.Vinfo) v2.getSPobject(); // apparently each member may have seperate byte order (!!!??) if (vm.typeInfo.endian >= 0) m.setDataObject(vm.typeInfo.endian == RandomAccessFile.LITTLE_ENDIAN ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); // vm.dataPos : offset since start of Structure m.setDataParam((int) vm.dataPos); // track if there is a heap if (v2.getDataType() == DataType.STRING || v2.isVariableLength()) hasHeap = true; // recurse if (v2 instanceof Structure) { Structure nested = (Structure) v2; StructureMembers nestSm = nested.makeStructureMembers(); m.setStructureMembers(nestSm); hasHeap |= convertStructure(nested, nestSm); } } return hasHeap; }
H5iosp.setDebugFlags(new DebugFlagsImpl("HdfEos/turnOff")); DebugFlags debugFlags = new DebugFlagsImpl("HdfEos/turnOff"); debugFlags.set("HdfEos/turnOff",false); H5iosp.setDebugFlags(debugFlags); return DecodeQualification.INTENDED; } else { DebugFlags debugFlags = new DebugFlagsImpl("HdfEos/turnOff"); debugFlags.set("HdfEos/turnOff",false); H5iosp.setDebugFlags(debugFlags); if (ncfile != null) { try {
public static NetcdfFile open(Object input) throws IOException { byte[] buffer = new byte[MAGIC_BUFFER_LENGTH]; readMagicBytes(input, buffer); IOServiceProvider spi = getIOSpi(buffer); if (spi != null) { RandomAccessFile raf = getRaf(input); if (raf != null) { return new DummyNetcdfFile(spi, raf, raf.getLocation()); } } else { // The HDF5 superblock, which begins with the 8-byte format signature, // may begin at certain predefined offsets within the HDF5 file: // 0, 512, 1024, 2048, and multiples of two thereafter. // see: http://www.hdfgroup.org/HDF5/doc/H5.format.html RandomAccessFile rafForTesting = getRafForTesting(input); boolean isValidHdf5 = H5header.isValidFile(rafForTesting); if (isValidHdf5) { RandomAccessFile raf = getRaf(input); if (raf != null) { return new DummyNetcdfFile(new H5iosp(), raf, raf.getLocation()); } } else { if (!(input instanceof ImageInputStream)) { rafForTesting.close(); } } } return null; }
long address = chunk.getSrcPos() + layout.getElemSize() * i; Array vlenArray = headerParser.getHeapDataArray(address, readType, endian); data[count++] = (typeInfo.base.hdfType == 7) ? convertReference(vlenArray) : vlenArray; return convertStructure((Structure) v, layout, shape, byteArray); // LOOK return readDataPrimitive(layout, dataType, shape, fillValue, endian, true);
int destPos = 0; for (int i = 0; i< layout.getTotalNelems(); i++) { // loop over each structure h5iosp.convertHeap(asbb, destPos, sm); destPos += layout.getElemSize(); Array vlenArray = getHeapDataArray(address, readType, endian); if (vinfo.typeInfo.base.hdfType == 7) data[count++] = h5iosp.convertReference(vlenArray); else data[count++] = vlenArray; Object data = h5iosp.readDataPrimitive(layout, dataType, shape, null, endian, false); Array dataArray = null;
private ArrayStructure convertStructure(Structure s, Layout layout, int[] shape, byte[] byteArray) throws IOException, InvalidRangeException { // create StructureMembers - must set offsets StructureMembers sm = s.makeStructureMembers(); int calcSize = ArrayStructureBB.setOffsets(sm); // standard //ArrayStructureBB.showOffsets(sm, new Indent(2), new Formatter(System.out)); // special offset setting boolean hasHeap = convertStructure(s, sm); //ArrayStructureBB.showOffsets(sm, new Indent(2), new Formatter(System.out)); int recSize = layout.getElemSize(); if (recSize < calcSize) { log.error("calcSize = %d actualSize = %d%n", calcSize, recSize); throw new IOException("H5iosp illegal structure size " + s.getFullName()); } sm.setStructureSize(recSize); // place data into an ArrayStructureBB ByteBuffer bb = ByteBuffer.wrap(byteArray); ArrayStructureBB asbb = new ArrayStructureBB(sm, shape, bb, 0); // strings are stored on the heap, and must be read separately if (hasHeap) { int destPos = 0; for (int i = 0; i < layout.getTotalNelems(); i++) { // loop over each structure convertHeap(asbb, destPos, sm); destPos += layout.getElemSize(); } } return asbb; }
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { H5header.Vinfo vinfo = (H5header.Vinfo) v2.getSPobject(); if (debugRead) System.out.printf("%s read %s%n", v2.getFullName(), section); return readData(v2, vinfo.dataPos, section); }