public boolean conformal(Section other) { if (computeSize() != other.computeSize()) return false; Section reduceThis = this.reduce(); Section reduceOther = other.reduce(); return reduceThis.equalShape(reduceOther); }
Cache(int[] shape, int[] newshape, DataType dataType) { this.shape = shape; this.newshape = newshape; this.result = Array.factory(dataType, newshape); nt = shape[0]; Section s = new Section(shape); chunksize = (int)(s.computeSize() / nt); // get view of result as a 2d array (any..., nt); int[] reshape = new int[] {chunksize, nt}; this.work = this.result.reshapeNoCopy(reshape); }
public boolean equalShape(Section other) { if (computeSize() != other.computeSize()) return false; if (getRank() != other.getRank()) return false; for (int i = 0; i < getRank(); i++) { Range r = getRange(i); Range or = other.getRange(i); if (r.length() != or.length()) return false; } return true; }
public int[] computeUnlimitedChunking(List<Dimension> dims, int elemSize) { int maxElements = defaultChunkSize / elemSize; int[] result = fillRightmost(convertUnlimitedShape(dims), maxElements); long resultSize = new Section(result).computeSize(); if (resultSize < minChunksize) { maxElements = minChunksize / elemSize; result = incrUnlimitedShape(dims, result, maxElements); } return result; }
public int[] computeUnlimitedChunking(List<Dimension> dims, int elemSize) { int maxElements = defaultChunkSize / elemSize; int[] result = fillRightmost(convertUnlimitedShape(dims), maxElements); long resultSize = new Section(result).computeSize(); if (resultSize < minChunksize) { maxElements = minChunksize / elemSize; result = incrUnlimitedShape(dims, result, maxElements); } return result; }
/** * Make a section of an ArrayStructureBB * * @param org the original * @param section of the whole * @return original, if section is empty or the same saze as the original, else a section of the original */ static public ArrayStructureBB factory(ArrayStructureBB org, Section section) { if (section == null || section.computeSize() == org.getSize()) return org; return new ArrayStructureBBsection(org.getStructureMembers(), org.getShape(), org.getByteBuffer(), section); }
@Override public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { // see if its really a full read if ((null == section) || section.computeSize() == getSize()) return reallyRead(client, cancelTask); if (orgVar == null) return getMissingDataArray(section.getShape()); return orgVar.read(section); }
@Override public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException { Vinfo vinfo = (Vinfo) v2.getSPobject(); int vlen = (int) v2.getSize(); int len = (int) section.computeSize(); if (vlen == len) // entire array return readDataAll(vinfo.g4.grpid, vinfo.varid, vinfo.typeid, v2.getShapeAsSection()); //if(!section.isStrided()) // optimisation for unstrided section // return readUnstrided(vinfo.grpid, vinfo.varid, vinfo.typeid, section); return readDataSection(vinfo.g4.grpid, vinfo.varid, vinfo.typeid, section); }
@Override public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { // see if its really a full read if ((null == section) || section.computeSize() == getSize()) return reallyRead(client, cancelTask); if (orgVar == null) return getMissingDataArray(section.getShape()); return orgVar.read(section); }
@Override public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException { Vinfo vinfo = (Vinfo) v2.getSPobject(); int vlen = (int) v2.getSize(); int len = (int) section.computeSize(); if (vlen == len) // entire array return readDataAll(vinfo.grpid, vinfo.varid, vinfo.typeid, v2.getShapeAsSection()); //if (!section.isStrided()) // optimisation for unstrided section // return readUnstrided(vinfo.grpid, vinfo.varid, vinfo.typeid, section); return readDataSection(vinfo.grpid, vinfo.varid, vinfo.typeid, section); }
@Override public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException { Vinfo vinfo = (Vinfo) v2.getSPobject(); int vlen = (int) v2.getSize(); int len = (int) section.computeSize(); if (vlen == len) // entire array return readDataAll(vinfo.g4.grpid, vinfo.varid, vinfo.typeid, v2.getShapeAsSection()); //if(!section.isStrided()) // optimisation for unstrided section // return readUnstrided(vinfo.grpid, vinfo.varid, vinfo.typeid, section); return readDataSection(vinfo.g4.grpid, vinfo.varid, vinfo.typeid, section); }
@Override public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { // see if its really a full read if ((null == section) || section.computeSize() == getSize()) return reallyRead(client, cancelTask); if (orgVar == null) return getMissingDataArray(section.getShape()); return orgVar.read(section); }
DataStorage(int size, long filePos, NcStreamProto.Data dproto) { this.size = size; this.filePos = filePos; section = NcStream.decodeSection(dproto.getSection()); nelems = (int) section.computeSize(); bo = NcStream.decodeDataByteOrder(dproto); isVlen = dproto.getVdata(); isDeflate = dproto.getCompress() == NcStreamProto.Compress.DEFLATE; if (isDeflate) uncompressedLen = dproto.getUncompressedSize(); }
@Override protected Array _read(Section section) throws IOException, InvalidRangeException { // really a full read if ((null == section) || section.computeSize() == getSize()) return _read(); Array result; if (hasCachedData()) result = super._read(section); else result = proxyReader.reallyRead(this, section, null); return convert(result); }
private void testRealStrategy(Dimension[] shape, int elemSize) { List<Dimension> dims = Arrays.asList(shape); show("shape", dims); System.out.printf(" elemSize = %d%n", elemSize); Nc4ChunkingDefault chunker = new Nc4ChunkingDefault(); int[] result = chunker.computeUnlimitedChunking(dims, elemSize); show("chunk", result); long shapeSize = new Section(result).computeSize(); System.out.printf(" size = %d%n%n", shapeSize); int expectSize = chunker.getMinChunksize() / elemSize; assert shapeSize == expectSize : shapeSize +" != "+ expectSize; }
private void testOne(int[] shape, long maxChunkElems) { show("shape", shape); System.out.printf(" max = %d%n", maxChunkElems); FileWriter2.ChunkingIndex index = new FileWriter2.ChunkingIndex(shape); int[] result = index.computeChunkShape(maxChunkElems); show("chunk", result); long shapeSize = new Section(result).computeSize(); System.out.printf(" size = %d%n%n", shapeSize); assert shapeSize <= maxChunkElems; }
@Test public void testPart() throws InvalidRangeException { int[] full = new int[] {2, 10, 20}; int[] part = new int[] {2, 5, 20}; Section section = new Section(part); IndexChunker index = new IndexChunker(full, section); assert index.getTotalNelems() == section.computeSize(); IndexChunker.Chunk chunk = index.next(); assert chunk.getNelems() == section.computeSize()/2; }
@Test public void testFull() throws InvalidRangeException { int[] shape = new int[] {123,22,92,12}; Section section = new Section(shape); IndexChunker index = new IndexChunker(shape, section); assert index.getTotalNelems() == section.computeSize(); IndexChunker.Chunk chunk = index.next(); assert chunk.getNelems() == section.computeSize(); assert !index.hasNext(); }
public Array readData(Variable v2, Section wantSection) throws IOException, InvalidRangeException { // raf.seek(0); raf.order(RandomAccessFile.BIG_ENDIAN); int size = (int) wantSection.computeSize(); short[] arr = new short[size]; LayoutRegular indexer = new LayoutRegular(0, v2.getElementSize(), v2.getShape(), wantSection); while (indexer.hasNext()) { Layout.Chunk chunk = indexer.next(); raf.seek(chunk.getSrcPos()); raf.readShort(arr, (int) chunk.getDestElem(), chunk.getNelems()); // copy into primitive array } return Array.factory(v2.getDataType().getPrimitiveClassType(), wantSection.getShape(), arr); }
public Array readData(Variable v2, Section wantSection) throws IOException, InvalidRangeException { // raf.seek(0); raf.order(RandomAccessFile.BIG_ENDIAN); int size = (int) wantSection.computeSize(); short[] arr = new short[size]; LayoutRegular indexer = new LayoutRegular(0, v2.getElementSize(), v2.getShape(), wantSection); while (indexer.hasNext()) { Layout.Chunk chunk = indexer.next(); raf.seek(chunk.getSrcPos()); raf.readShort(arr, (int) chunk.getDestElem(), chunk.getNelems()); // copy into primitive array } return Array.factory(v2.getDataType().getPrimitiveClassType(), wantSection.getShape(), arr); }