TimeseriesIterator() { varIters = new ArrayList<>(); for (VarData vd : varData) { Array data = vd.array.getData(); if (debug) System.out.printf("%s shape=%s%n", vd.cov.getName(), Misc.showInts(data.getShape())); varIters.add(new VarIter(vd.cov, vd.array, data.getIndexIterator())); nvalues = (int) data.getSize(); if (timeAxis == null) { // assume they are all the same (!) CoverageCoordSys csys = vd.array.getCoordSysForData(); timeAxis = (CoverageCoordAxis1D) csys.getTimeAxis(); // LOOK may not be right } } }
public CdmrFeatureProto.GeoReferencedArray.Builder encodeGeoReferencedArray(GeoReferencedArray geoArray, boolean deflate) { CdmrFeatureProto.GeoReferencedArray.Builder builder = CdmrFeatureProto.GeoReferencedArray.newBuilder(); builder.setCoverageName(geoArray.getCoverageName()); builder.setDataType(NcStream.convertDataType(geoArray.getDataType())); builder.setVersion(3); // set to >= 3 for proto3 builder.setBigend(ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); int shape[] = geoArray.getData().getShape(); for (int aShape : shape) builder.addShape(aShape); CoverageCoordSys csys = geoArray.getCoordSysForData(); for (String axisName : csys.getAxisNames()) // geoArray.getAxisNames()) // LOOK could use csys.getAxisNames(), but order may be incorrect, must match shape builder.addAxisName(axisName); builder.setCoordSysName(csys.getName()); if (deflate) { builder.setCompress(NcStreamProto.Compress.DEFLATE); long uncompressedSize = geoArray.getData().getSizeBytes(); builder.setUncompressedSize(uncompressedSize); } else { builder.setPrimdata(NcStreamDataCol.copyArrayToByteString( geoArray.getData())); } return builder; }
private static boolean readRandom(FeatureDatasetCoverage covDataset, PrintWriter pw ) throws IOException, InvalidRangeException { CoverageCollection cc = covDataset.getCoverageCollections().get(0); int ncov = cc.getCoverageCount(); Random r = new Random(System.currentTimeMillis()); int randomIdx = r.nextInt(ncov); int count = 0; Coverage randomCov = null; for (Coverage c : cc.getCoverages()) { if (count == randomIdx) { randomCov = c; break; } count++; } if (randomCov == null) { pw.format("Bad random coverage"); return false; } SubsetParams subset = new SubsetParams().setTimePresent(); GeoReferencedArray geo = randomCov.readData(subset); Array data = geo.getData(); System.out.printf(" read data from %s shape = %s%n", randomCov.getName(), Misc.showInts(data.getShape())); return true; }
private void checkConformance(Coverage gridSubset, GeoReferencedArray geo, String where) { CoverageCoordSys csys = gridSubset.getCoordSys(); CoverageCoordSys csysData = geo.getCoordSysForData(); //System.out.printf(" csys=%s%n", csys); //System.out.printf("csysData=%s%n", csysData); Section s = new Section(csys.getShape()); Section so = new Section(csysData.getShape()); boolean ok = s.conformal(so); int[] dataShape = geo.getData().getShape(); //System.out.printf("dataShape=%s%n", Misc.showInts(dataShape)); Section sdata = new Section(dataShape); boolean ok2 = s.conformal(sdata); if (!ok || !ok2) logger.warn("CFGridCoverageWriter2 checkConformance fails " +where); } }
private void writeCoverageData(CoverageCollection gdsOrg, SubsetParams subsetParams, CoverageCollection subsetDataset, NetcdfFileWriter writer) throws IOException, InvalidRangeException { for (Coverage coverage : subsetDataset.getCoverages()) { // we need to call readData on the original Coverage coverageOrg = gdsOrg.findCoverage(coverage.getName()); GeoReferencedArray array = coverageOrg.readData(subsetParams); // test conform to whatever axis.getCoordsAsArray() returns checkConformance(coverage, array, gdsOrg.getName()); Variable v = writer.findVariable(coverage.getName()); if (show) System.out.printf("CFGridCoverageWriter2 write coverage %s%n", v.getNameAndDimensions()); writer.write(v, array.getData()); } }
static int timeDataRead2(String remote, int stride) throws IOException, InvalidRangeException { logger.debug("--Coverage Read {} stride={} ", remote, stride); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(remote)) { CoverageCollection gcs = cc.getSingleCoverageCollection(); String gridName = "10_metre_V_wind_component_surface"; Coverage grid = gcs.findCoverage(gridName); Assert.assertNotNull(gridName, grid); SubsetParams params = new SubsetParams().setTimePresent().setHorizStride(stride); long start = System.currentTimeMillis(); GeoReferencedArray geo = grid.readData(params); long took = System.currentTimeMillis() - start; logger.debug("took={} size={}", took, geo.getData().getSize()); } return 1; } }
private void checkLatLonSubset(CoverageCollection gcs, Coverage coverage, LatLonRect bbox, int[] expectedShape) throws Exception { System.out.printf(" coverage llbb = %s width=%f%n", gcs.getLatlonBoundingBox().toString2(), gcs.getLatlonBoundingBox().getWidth()); System.out.printf(" constrain bbox= %s width=%f%n", bbox.toString2(), bbox.getWidth()); SubsetParams params = new SubsetParams().setLatLonBoundingBox(bbox).setTimePresent(); GeoReferencedArray geo = coverage.readData(params); CoverageCoordSys gcs2 = geo.getCoordSysForData(); Assert.assertNotNull("CoordSysForData", gcs2); System.out.printf(" data cs shape=%s%n", Misc.showInts(gcs2.getShape())); System.out.printf(" data shape=%s%n", Misc.showInts(geo.getData().getShape())); Assert.assertArrayEquals("CoordSys=Data shape", gcs2.getShape(), geo.getData().getShape()); Assert.assertArrayEquals("expected data shape", expectedShape, geo.getData().getShape()); }
@Test public void testBestTimeOffsetCoord() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4"; String covName = "Temperature_altitude_above_msl"; logger.debug("testBestPresent Dataset {} coverage {}", endpoint, covName); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, cc); CoverageCollection gcs = cc.findCoverageDataset(FeatureType.GRID); Assert.assertNotNull("gcs", gcs); Coverage cover = gcs.findCoverage(covName); Assert.assertNotNull(covName, cover); SubsetParams params = new SubsetParams(); params.setTimeOffset(48.0); params.setVertCoord(3658.0); logger.debug(" subset {}", params); GeoReferencedArray geo = cover.readData(params); // should not be missing ! assert !Float.isNaN(geo.getData().getFloat(0)); Array data = geo.getData(); Index ai = data.getIndex(); float testValue = data.getFloat(ai.set(0, 0, 3, 0)); Assert2.assertNearlyEquals(250.5, testValue); } }
@Test public void testSrcNoParams() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "ncss/GFS/CONUS_80km/GFS_CONUS_80km_20120227_0000.grib1"; String covName = "Temperature_isobaric"; logger.debug("testSrcNoParams Dataset {} coverage {}", endpoint, covName); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, cc); CoverageCollection gcs = cc.findCoverageDataset(FeatureType.GRID); Assert.assertNotNull("gcs", gcs); Coverage cover = gcs.findCoverage(covName); Assert.assertNotNull(covName, cover); SubsetParams params = new SubsetParams(); logger.debug(" subset {}", params); GeoReferencedArray geo = cover.readData(params); int[] resultShape = geo.getData().getShape(); int[] expectShape = new int[] {36, 29, 65, 93}; Assert.assertArrayEquals("shape", expectShape, resultShape); } }
@Test // there is no interval with offset value = 51 public void testNoIntervalFound() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4"; String covName = "Momentum_flux_u-component_surface_Mixed_intervals_Average"; logger.debug("test1Runtime1TimeOffset Dataset {} coverage {}", endpoint, covName); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, cc); CoverageCollection gcs = cc.findCoverageDataset(FeatureType.FMRC); Assert.assertNotNull("gcs", gcs); Coverage cover = gcs.findCoverage(covName); Assert.assertNotNull(covName, cover); SubsetParams params = new SubsetParams(); CalendarDate runtime = CalendarDate.parseISOformat(null, "2015-03-01T12:00:00Z"); params.set(SubsetParams.runtime, runtime); double offsetVal = 51.0; // should fail params.set(SubsetParams.timeOffset, offsetVal); logger.debug(" subset {}", params); GeoReferencedArray geo = cover.readData(params); testGeoArray(geo, runtime, null, offsetVal); // should be empty, but instead its a bunch of NaNs assert Float.isNaN(geo.getData().getFloat(0)); } }
@Test public void TestCFWriterCoverageRead() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "ncss/test/GFS_CONUS_80km_20120227_0000.grib1.nc4"; logger.info("open {}", endpoint); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { assert cc != null; Assert.assertEquals(1, cc.getCoverageCollections().size()); CoverageCollection gds = cc.getCoverageCollections().get(0); Assert.assertNotNull(endpoint, gds); Assert.assertEquals(FeatureType.FMRC, gds.getCoverageType()); String covName = "Temperature_isobaric"; Coverage cover = gds.findCoverage(covName); Assert.assertNotNull(covName, cover); long size = cover.getSizeInBytes(); Assert.assertEquals(25243920, size); SubsetParams subset = new SubsetParams().setVertCoord(300.0).setTimeOffset(42.0); GeoReferencedArray geo = cover.readData(subset); Array data = geo.getData(); logger.info("{}", Misc.showInts(data.getShape())); } } }
private void readCoverageData(Coverage cover, SubsetParams coords) throws IOException, InvalidRangeException { countReadsForVariable++; if (MAX_READS > 0 && countReadsForVariable > MAX_READS) return; GribDataReader.currentDataRecord = null; GeoReferencedArray geoArray = cover.readData(coords); int[] shape = geoArray.getData().getShape(); if (shape[1] != 1) { cover.readData(coords); } for (int i = 0; i < shape.length - 2; i++) { Assert.assertEquals(Misc.showInts(shape), 1, shape[i]); } if (isGrib1) readAndTestGrib1(cover.getName(), coords); else readAndTestGrib2(cover.getName(), coords); this.cover = null; }
@Test public void testBadSize() throws IOException, InvalidRangeException { String covName = "Geopotential_height_isobaric"; Catalog cat = TdsLocalCatalog.open("catalog/rdaTest/ds084.3/catalog.xml"); Assert.assertNotNull(cat); Dataset ds = cat.findDatasetByID("rdaTest/ds084.3/TwoD"); DataFactory fac = new DataFactory(); try ( DataFactory.Result result = fac.openFeatureDataset(ds, null)) { Assert.assertFalse(result.errLog.toString(), result.fatalError); Assert.assertNotNull(result.featureDataset); Assert.assertEquals(FeatureDatasetCoverage.class, result.featureDataset.getClass()); FeatureDatasetCoverage fdc = (FeatureDatasetCoverage) result.featureDataset; CoverageCollection cc = fdc.findCoverageDataset(FeatureType.FMRC); Assert.assertNotNull(FeatureType.FMRC.toString(), cc); Coverage cov = cc.findCoverage(covName); Assert.assertNotNull(covName, cov); SubsetParams subset = new SubsetParams().setTimePresent(); GeoReferencedArray geo = cov.readData(subset); Array data = geo.getData(); System.out.printf(" read data from %s shape = %s%n", cov.getName(), Misc.showInts(data.getShape())); } } }
@Test // 1 runtime, 1 time (Time2DCoordSys case 1b) public void test1Runtime1TimeIntervalEdge() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4"; String covName = "Momentum_flux_u-component_surface_Mixed_intervals_Average"; logger.debug("test1Runtime1TimeInterval Dataset {} coverage {}", endpoint, covName); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, cc); CoverageCollection gcs = cc.findCoverageDataset(FeatureType.FMRC); Assert.assertNotNull("gcs", gcs); Coverage cover = gcs.findCoverage(covName); Assert.assertNotNull(covName, cover); SubsetParams params = new SubsetParams(); CalendarDate runtime = CalendarDate.parseISOformat(null, "2015-03-01T00:00:00Z"); params.set(SubsetParams.runtime, runtime); CalendarDate time = CalendarDate.parseISOformat(null, "2015-03-06T19:30:00Z"); // (6,12), (12,18) params.set(SubsetParams.time, time); logger.debug(" subset {}", params); GeoReferencedArray geo = cover.readData(params); testGeoArray(geo, runtime, time, null); Array data = geo.getData(); Index ai = data.getIndex(); float testValue = data.getFloat(ai.set(0, 0, 3, 0)); Assert2.assertNearlyEquals(0.244f, testValue); } }
@Test // 1 runtime, 1 time (Time2DCoordSys case 1b) public void test1Runtime1Time() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4"; String covName = "Total_ozone_entire_atmosphere_single_layer"; logger.debug("test1Runtime1Time Dataset {} coverage {}", endpoint, covName); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, cc); CoverageCollection gcs = cc.findCoverageDataset(FeatureType.FMRC); Assert.assertNotNull("gcs", gcs); Coverage cover = gcs.findCoverage(covName); Assert.assertNotNull(covName, cover); SubsetParams params = new SubsetParams(); CalendarDate runtime = CalendarDate.parseISOformat(null, "2015-03-01T06:00:00Z"); params.set(SubsetParams.runtime, runtime); CalendarDate time = CalendarDate.parseISOformat(null, "2015-03-01T12:00:00Z"); params.set(SubsetParams.time, time); logger.debug(" subset {}", params); GeoReferencedArray geo = cover.readData(params); testGeoArray(geo, runtime, time, null); Array data = geo.getData(); Index ai = data.getIndex(); float testValue = data.getFloat(ai.set(0,0,1,0)); Assert2.assertNearlyEquals(371.5, testValue); } }
public void testWrongRuntime() throws IOException, InvalidRangeException { Grib.setDebugFlags(new DebugFlagsImpl("Grib/debugGbxIndexOnly")); String endpoint = "D:/work/rdavm/index/ds083.2_Grib2.ncx4"; String covName = "Soil_temperature_depth_below_surface_layer"; try (FeatureDatasetCoverage fdc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, fdc); CoverageCollection cc = fdc.findCoverageDataset(FeatureType.GRID); Assert.assertNotNull(FeatureType.GRID.toString(), cc); Coverage cov = cc.findCoverage(covName); Assert.assertNotNull(covName, cov); SubsetParams subset = new SubsetParams().setTimePresent(); GeoReferencedArray geo = cov.readData(subset); Array data = geo.getData(); System.out.printf(" read data from %s shape = %s%n", cov.getName(), Misc.showInts(data.getShape())); } Grib.setDebugFlags(new DebugFlagsImpl("")); }
@Test public void testBadSize() throws IOException, InvalidRangeException { // Grib.setDebugFlags(new DebugFlagsImpl("Grib/debugGbxIndexOnly")); String endpoint = "D:/work/rdavm/ds084.3/ds084.3.ncx4"; String covName = "Geopotential_height_isobaric"; try (FeatureDatasetCoverage fdc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, fdc); CoverageCollection cc = fdc.findCoverageDataset(FeatureType.GRID); Assert.assertNotNull(FeatureType.FMRC.toString(), cc); Coverage cov = cc.findCoverage(covName); Assert.assertNotNull(covName, cov); SubsetParams subset = new SubsetParams().setTimePresent(); GeoReferencedArray geo = cov.readData(subset); Array data = geo.getData(); System.out.printf(" read data from %s shape = %s%n", cov.getName(), Misc.showInts(data.getShape())); } Grib.setDebugFlags(new DebugFlagsImpl("")); }
@Test public void TestNetcdfCurvilinear() throws IOException, InvalidRangeException { String endpoint = TestDir.cdmUnitTestDir + "ft/coverage/Run_20091025_0000.nc"; // NetCDF has 2D and 1D logger.debug("open {}", endpoint); try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) { assert cc != null; Assert.assertEquals(1, cc.getCoverageCollections().size()); CoverageCollection gds = cc.getCoverageCollections().get(0); Assert.assertNotNull(endpoint, gds); Assert.assertEquals(FeatureType.CURVILINEAR, gds.getCoverageType()); Assert.assertEquals(20, gds.getCoverageCount()); String covName = "u"; Coverage cover = gds.findCoverage(covName); Assert.assertNotNull(covName, cover); SubsetParams params = new SubsetParams().setVertCoord(-.05).set(SubsetParams.timePresent, true); GeoReferencedArray geo = cover.readData(params); Array data = geo.getData(); Index ima = data.getIndex(); int[] expectedShape = new int[] {1,1,22,12}; Assert.assertArrayEquals(expectedShape, data.getShape()); Assert2.assertNearlyEquals(0.0036624447, data.getDouble(ima.set(0, 0, 0, 0)), 1e-6); Assert2.assertNearlyEquals(0.20564626, data.getDouble(ima.set(0, 0, 21, 11)), 1e-6); } }
@Test public void testNonOrthMRUTC() throws IOException, InvalidRangeException { Grib.setDebugFlags(new DebugFlagsImpl("Grib/debugGbxIndexOnly")); String endpoint = "D:/work/rdavm/ds277.6/monthly/ds277.6.ncx4"; String ccName = "ds277.6#MRUTC-LatLon_418X360-4p83S-179p50W"; String covName = "Salinity_depth_below_sea_Average"; try (FeatureDatasetCoverage fdc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, fdc); CoverageCollection cc = fdc.findCoverageDataset(ccName); Assert.assertNotNull(ccName, cc); Coverage cov = cc.findCoverage(covName); Assert.assertNotNull(covName, cov); SubsetParams subset = new SubsetParams().setTimePresent(); GeoReferencedArray geo = cov.readData(subset); Array data = geo.getData(); System.out.printf(" read data from %s shape = %s%n", cov.getName(), Misc.showInts(data.getShape())); } Grib.setDebugFlags(new DebugFlagsImpl("")); }
@Test public void testNegDataSize() throws IOException, InvalidRangeException { Grib.setDebugFlags(new DebugFlagsImpl("Grib/debugGbxIndexOnly")); String endpoint = "D:/work/rdavm/ds084.3/ds084.3.ncx4"; String covName = "v-component_of_wind_potential_vorticity_surface"; try (FeatureDatasetCoverage fdc = CoverageDatasetFactory.open(endpoint)) { Assert.assertNotNull(endpoint, fdc); CoverageCollection cc = fdc.findCoverageDataset(FeatureType.GRID); Assert.assertNotNull(FeatureType.FMRC.toString(), cc); Coverage cov = cc.findCoverage(covName); Assert.assertNotNull(covName, cov); SubsetParams subset = new SubsetParams().setTimePresent(); GeoReferencedArray geo = cov.readData(subset); Array data = geo.getData(); System.out.printf(" read data from %s shape = %s%n", cov.getName(), Misc.showInts(data.getShape())); } Grib.setDebugFlags(new DebugFlagsImpl("")); }