private void readIndex(String indexFilename) throws IOException { try (FileInputStream fin = new FileInputStream(indexFilename)) { if (!NcStream.readAndTest(fin, MAGIC_START.getBytes(CDM.utf8Charset))) throw new IllegalStateException("bad index file"); int version = fin.read(); if (version != 1) throw new IllegalStateException("Bad version = " + version); int count = NcStream.readVInt(fin); for (int i = 0; i < count; i++) { int size = NcStream.readVInt(fin); byte[] pb = new byte[size]; NcStream.readFully(fin, pb); StationIndex si = decodeStationIndex(pb); map.put(si.stnId, si); } } System.out.println(" read index map size=" + map.values().size()); }
private void readIndex(String indexFilename) throws IOException { FileInputStream fin = new FileInputStream(indexFilename); if (!NcStream.readAndTest(fin, MAGIC_START.getBytes(CDM.utf8Charset))) throw new IllegalStateException("bad index file"); int version = fin.read(); if (version != 1) throw new IllegalStateException("Bad version = " + version); int count = NcStream.readVInt(fin); for (int i = 0; i < count; i++) { int size = NcStream.readVInt(fin); byte[] pb = new byte[size]; NcStream.readFully(fin, pb); StationIndex si = decodeStationIndex(pb); map.put(si.stnId, si); } fin.close(); System.out.println(" read index map size=" + map.values().size()); }
private void readIndex(String indexFilename) throws IOException { try (FileInputStream fin = new FileInputStream(indexFilename)) { if (!NcStream.readAndTest(fin, MAGIC_START_IDX.getBytes(CDM.utf8Charset))) throw new IllegalStateException("bad index file"); int version = fin.read(); if (version != 1) throw new IllegalStateException("Bad version = " + version); int count = NcStream.readVInt(fin); for (int i = 0; i < count; i++) { int size = NcStream.readVInt(fin); byte[] pb = new byte[size]; NcStream.readFully(fin, pb); StationIndex si = decodeStationIndex(pb); map.put(si.stnId, si); } } System.out.println(" read index map size=" + map.values().size()); }
private void readIndex(String indexFilename) throws IOException { try (FileInputStream fin = new FileInputStream(indexFilename)) { if (!NcStream.readAndTest(fin, MAGIC_START.getBytes(CDM.utf8Charset))) throw new IllegalStateException("bad index file"); int version = fin.read(); if (version != 1) throw new IllegalStateException("Bad version = " + version); int count = NcStream.readVInt(fin); for (int i = 0; i < count; i++) { int size = NcStream.readVInt(fin); byte[] pb = new byte[size]; NcStream.readFully(fin, pb); StationIndex si = decodeStationIndex(pb); map.put(si.stnId, si); } } System.out.println(" read index map size=" + map.values().size()); }
private void readIndex(String indexFilename) throws IOException { FileInputStream fin = new FileInputStream(indexFilename); if (!NcStream.readAndTest(fin, MAGIC_START_IDX.getBytes(CDM.utf8Charset))) throw new IllegalStateException("bad index file"); int version = fin.read(); if (version != 1) throw new IllegalStateException("Bad version = "+version); int count = NcStream.readVInt(fin); for (int i=0; i<count; i++) { int size = NcStream.readVInt(fin); byte[] pb = new byte[size]; NcStream.readFully(fin, pb); StationIndex si = decodeStationIndex(pb); map.put(si.stnId, si); } fin.close(); System.out.println(" read index map size=" + map.values().size()); }
private void readIndex(String indexFilename) throws IOException { try (FileInputStream fin = new FileInputStream(indexFilename)) { if (!NcStream.readAndTest(fin, MAGIC_START_IDX.getBytes(CDM.utf8Charset))) throw new IllegalStateException("bad index file"); int version = fin.read(); if (version != 1) throw new IllegalStateException("Bad version = " + version); int count = NcStream.readVInt(fin); for (int i = 0; i < count; i++) { int size = NcStream.readVInt(fin); byte[] pb = new byte[size]; NcStream.readFully(fin, pb); StationIndex si = decodeStationIndex(pb); map.put(si.stnId, si); } } System.out.println(" read index map size=" + map.values().size()); }
public GeoReferencedArray readData(CoverageDataResponse dataResponse, GeoArrayResponse arrayResponse, InputStream is) throws IOException { int sizeIn = NcStream.readVInt(is); // not used ? if (arrayResponse.deflate) { is = new InflaterInputStream(is); float ratio = (sizeIn == 0) ? 0.0f : ((float) arrayResponse.uncompressedSize) / sizeIn; if (showCompression) System.out.printf(" readData data message compress= %d decompress=%d compress=%f%n", sizeIn, arrayResponse.uncompressedSize, ratio); } byte[] datab = new byte[(int) arrayResponse.uncompressedSize]; NcStream.readFully(is, datab); Array data = Array.factory(arrayResponse.dataType, arrayResponse.shape, ByteBuffer.wrap(datab)); CoverageCoordSys csys = dataResponse.findCoordSys( arrayResponse.coordSysName); if (csys == null) throw new IOException("Misformed response - no coordsys"); return new GeoReferencedArray(arrayResponse.coverageName, arrayResponse.dataType, data, csys); }
private void readNext() throws IOException { byte[] b = new byte[4]; NcStream.readFully(is, b); // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER if (test(b, NcStream.MAGIC_VDATA)) { int dsize = NcStream.readVInt(is); byte[] datab = new byte[dsize]; NcStream.readFully(is, datab); curr = NcStream.decodeStructureData(members, datab); // System.out.printf("StreamDataIterator read sdata size= %d%n", dsize); } else if (test(b, NcStream.MAGIC_VEND)) { curr = null; } else { throw new IllegalStateException("bad stream"); } }
private void readNext() throws IOException { byte[] b = new byte[4]; NcStream.readFully(is, b); // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER if (test(b, NcStream.MAGIC_VDATA)) { int dsize = NcStream.readVInt(is); byte[] datab = new byte[dsize]; NcStream.readFully(is, datab); curr = NcStream.decodeStructureData(members, datab); // System.out.printf("StreamDataIterator read sdata size= %d%n", dsize); } else if (test(b, NcStream.MAGIC_VEND)) { curr = null; } else { throw new IllegalStateException("bad stream"); } }
private void readNext() throws IOException { byte[] b = new byte[4]; NcStream.readFully(is, b); // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER if (NcStream.test(b, NcStream.MAGIC_VDATA)) { int dsize = NcStream.readVInt(is); byte[] datab = new byte[dsize]; NcStream.readFully(is, datab); // curr = NcStream.decodeStructureData(members, bo, datab); LOOK // System.out.printf("StreamDataIterator read sdata size= %d%n", dsize); } else if (NcStream.test(b, NcStream.MAGIC_VEND)) { curr = null; close(); } else { throw new IllegalStateException("bad stream"); } }
public PointFeatureIterator getPointFeatureIterator(int bufferSize) throws IOException { String query = PointDatasetRemote.makeQuery("stn=" + s.getName(), null, dateRange); InputStream in = null; try { in = CdmRemote.sendQuery(uri, query); PointStream.MessageType mtype = PointStream.readMagic(in); if (mtype == PointStream.MessageType.End) { // no obs were found in.close(); return new PointIteratorEmpty(); // return empty iterator } if (mtype != PointStream.MessageType.PointFeatureCollection) { throw new RuntimeException("Station Request: bad response = " + mtype); } int len = NcStream.readVInt(in); byte[] b = new byte[len]; NcStream.readFully(in, b); PointStreamProto.PointFeatureCollection pfc = PointStreamProto.PointFeatureCollection.parseFrom(b); riter = new RemotePointFeatureIterator(in, new PointStream.ProtobufPointFeatureMaker(pfc)); riter.setCalculateBounds(this); return riter; } catch (Throwable t) { if (in != null) in.close(); throw new IOException(t.getMessage(), t); } } }
@Override public PointFeatureIterator getPointFeatureIterator(int bufferSize) throws IOException { String query = PointDatasetRemote.makeQuery("stn=" + s.getName(), null, dateRange); InputStream in = null; try { in = CdmRemote.sendQuery(uri, query); PointStream.MessageType mtype = PointStream.readMagic(in); if (mtype == PointStream.MessageType.End) { // no obs were found in.close(); return new PointIteratorEmpty(); // return empty iterator } if (mtype != PointStream.MessageType.PointFeatureCollection) { throw new RuntimeException("Station Request: bad response = " + mtype); } int len = NcStream.readVInt(in); byte[] b = new byte[len]; NcStream.readFully(in, b); PointStreamProto.PointFeatureCollection pfc = PointStreamProto.PointFeatureCollection.parseFrom(b); riter = new RemotePointFeatureIterator(in, new PointStream.ProtobufPointFeatureMaker(pfc)); riter.setCalculateBounds(this); return riter; } catch (Throwable t) { if (in != null) in.close(); throw new IOException(t.getMessage(), t); } } }
private DataResult readData2(InputStream is) throws IOException { int psize = NcStream.readVInt(is); if (debug) System.out.println(" readData data message len= " + psize); byte[] dp = new byte[psize]; NcStream.readFully(is, dp); NcStreamProto.DataCol dproto = NcStreamProto.DataCol.parseFrom(dp); // NcStreamProto.Data2 dproto = NcStreamProto.Data2.parseDelimitedFrom(is); NcStreamDataCol decoder = new NcStreamDataCol(); Array data = decoder.decode(dproto, null); return new DataResult(dproto.getName(), data); }
@Override public PointFeatureIterator getPointFeatureIterator() throws IOException { String query = PointDatasetRemote.makeQuery("stn=" + s.getName(), null, getInfo().getCalendarDateRange(this.getTimeUnit())); InputStream in = null; try { in = CdmRemote.sendQuery(null, uri, query); PointStream.MessageType mtype = PointStream.readMagic(in); if (mtype == PointStream.MessageType.End) { // no obs were found in.close(); return new PointIteratorEmpty(); // return empty iterator } if (mtype != PointStream.MessageType.PointFeatureCollection) { throw new RuntimeException("Station Request: bad response = " + mtype); } int len = NcStream.readVInt(in); byte[] b = new byte[len]; NcStream.readFully(in, b); PointStreamProto.PointFeatureCollection pfc = PointStreamProto.PointFeatureCollection.parseFrom(b); riter = new PointIteratorStream(StationFeatureStream.this, in, new PointStream.ProtobufPointFeatureMaker(pfc)); return riter; } catch (Throwable t) { if (in != null) in.close(); throw new IOException(t.getMessage(), t); } } }
private boolean openIndex(RandomAccessFile indexRaf, Logger logger) { try { indexRaf.order(RandomAccessFile.BIG_ENDIAN); indexRaf.seek(0); //// header message magic = new byte[Grib2CollectionWriter.MAGIC_START.getBytes(CDM.utf8Charset).length]; // they are all the same indexRaf.readFully(magic); version = indexRaf.readInt(); long recordLength = indexRaf.readLong(); if (recordLength > Integer.MAX_VALUE) { logger.error("Grib2Collection {}: invalid recordLength size {}", indexRaf.getLocation(), recordLength); return false; } indexRaf.skipBytes(recordLength); int size = NcStream.readVInt(indexRaf); if ((size < 0) || (size > 100 * 1000 * 1000)) { logger.warn("GribCdmIndex {}: invalid index size {}", indexRaf.getLocation(), size); return false; } byte[] m = new byte[size]; indexRaf.readFully(m); gribCollectionIndex = GribCollectionProto.GribCollection.parseFrom(m); return true; } catch (Throwable t) { logger.error("Error reading index " + indexRaf.getLocation(), t); return false; } }
public StructureDataIterator getStructureIterator(InputStream is, NetcdfFile ncfile) throws IOException { if (!readAndTest(is, NcStream.MAGIC_DATA)) throw new IOException("Data transfer corrupted on "+ncfile.getLocation()); int psize = NcStream.readVInt(is); if (debug) System.out.println(" readData data message len= " + psize); byte[] dp = new byte[psize]; NcStream.readFully(is, dp); NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp); // if (debug) System.out.println(" readData proto = " + dproto); Structure s = (Structure) ncfile.findVariable(dproto.getVarName()); StructureMembers members = s.makeStructureMembers(); ArrayStructureBB.setOffsets(members); return new StreamDataIterator(is, members); }
public StructureDataIterator getStructureIterator(InputStream is, NetcdfFile ncfile) throws IOException { if (!readAndTest(is, NcStream.MAGIC_DATA)) throw new IOException("Data transfer corrupted on "+ncfile.getLocation()); int psize = NcStream.readVInt(is); if (debug) System.out.println(" readData data message len= " + psize); byte[] dp = new byte[psize]; NcStream.readFully(is, dp); NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp); // if (debug) System.out.println(" readData proto = " + dproto); Structure s = (Structure) ncfile.findVariable(dproto.getVarName()); StructureMembers members = s.makeStructureMembers(); ArrayStructureBB.setOffsets(members); return new StreamDataIterator(is, members); }
public StructureDataIterator getStructureIterator(InputStream is, NetcdfFile ncfile) throws IOException { if (!NcStream.readAndTest(is, NcStream.MAGIC_DATA)) throw new IOException("Data transfer corrupted on " + ncfile.getLocation()); int psize = NcStream.readVInt(is); if (debug) System.out.println(" readData data message len= " + psize); byte[] dp = new byte[psize]; NcStream.readFully(is, dp); NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp); // if (debug) System.out.println(" readData proto = " + dproto); Structure s = (Structure) ncfile.findVariable(dproto.getVarName()); StructureMembers members = s.makeStructureMembers(); ArrayStructureBB.setOffsets(members); ByteOrder bo = NcStream.decodeDataByteOrder(dproto); return new StreamDataIterator(is, members, bo); }
/** * initialize the stationHelper. */ @Override protected StationHelper createStationHelper() throws IOException { // read in all the stations with the "stations" query StationHelper stationHelper = new StationHelper(); try (InputStream in = CdmRemote.sendQuery(uri, "req=stations")) { PointStream.MessageType mtype = PointStream.readMagic(in); if (mtype != PointStream.MessageType.StationList) { throw new RuntimeException("Station Request: bad response"); } int len = NcStream.readVInt(in); byte[] b = new byte[len]; NcStream.readFully(in, b); PointStreamProto.StationList stationsp = PointStreamProto.StationList.parseFrom(b); for (ucar.nc2.ft.point.remote.PointStreamProto.Station sp : stationsp.getStationsList()) { // Station s = new StationImpl(sp.getId(), sp.getDesc(), sp.getWmoId(), sp.getLat(), sp.getLon(), sp.getAlt()); stationHelper.addStation(new RemoteStationFeatureImpl(null, null)); // LOOK WRONG } return stationHelper; } }
/** * initialize the stationHelper. */ @Override protected StationHelper createStationHelper() throws IOException { // read in all the stations with the "stations" query StationHelper stationHelper = new StationHelper(); try (InputStream in = CdmRemote.sendQuery(null, uri, "req=stations")) { PointStream.MessageType mtype = PointStream.readMagic(in); if (mtype != PointStream.MessageType.StationList) { throw new RuntimeException("Station Request: bad response"); } int len = NcStream.readVInt(in); byte[] b = new byte[len]; NcStream.readFully(in, b); PointStreamProto.StationList stationsp = PointStreamProto.StationList.parseFrom(b); for (ucar.nc2.ft.point.remote.PointStreamProto.Station sp : stationsp.getStationsList()) { // Station s = new StationImpl(sp.getId(), sp.getDesc(), sp.getWmoId(), sp.getLat(), sp.getLon(), sp.getAlt()); stationHelper.addStation(new StationFeatureStream(null, null)); // LOOK WRONG } return stationHelper; } }