@Override public CalendarDate extractRunDate(MFile mfile) { return dateExtractor.getCalendarDate(mfile); }
public Partition(MCollection dcm) { FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); if (config == null) logger.warn("Partition missing FeatureCollectionConfig {}", dcm); this.dcm = dcm; this.name = dcm.getCollectionName(); this.lastModified = dcm.getLastModified(); this.directory = StringUtil2.replace(dcm.getRoot(), '\\', "/"); this.partitionDate = dcm.getPartitionDate(); String indexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/"); if (partitionDate == null) { partitionDate = getDateExtractor().getCalendarDateFromPath(indexFilename); // LOOK dicey } // now remove the directory if (indexFilename.startsWith(directory)) { indexFilename = indexFilename.substring(directory.length()); if (indexFilename.startsWith("/")) indexFilename = indexFilename.substring(1); } filename = indexFilename; }
@Test public void testCalendarDateFromPath() throws IOException { Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser("[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America/FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm#.grib1", errlog); System.out.printf("errlog=%s%n", errlog); System.out.printf("specp=%s%n", specp); // test parsing assert specp.getRootDir().equals("[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America"); assert !specp.wantSubdirs(); assert specp.getFilter().toString().equals("FNMOC_COAMPS_Equatorial_America_..............grib1"); assert specp.getDateFormatMark().equals("FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm"); // test filter String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); // test date extractor DateExtractor extractor = new DateExtractorFromName(specp.getDateFormatMark(), true); CalendarDate cd = extractor.getCalendarDateFromPath(mfile.toString()); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }
@Override public CalendarDate extractDate(MFile mfile) { return (dateExtractor == null) ? null : dateExtractor.getCalendarDate(mfile); }
@Override public CalendarDate extractRunDate(MFile mfile) { return (dateExtractor == null) ? null : dateExtractor.getCalendarDate(mfile); }
@Override public CalendarDate extractDate(MFile mfile) { return (dateExtractor == null) ? null : dateExtractor.getCalendarDate(mfile); }
private List<CollectionManager> makePartitionsByPeriod() throws IOException { this.type = Type.timePeriod; List<DatedMFile> files = new ArrayList<DatedMFile>(); for (MFile mfile : getFiles()) { CalendarDate cdate = dateExtractor.getCalendarDate(mfile); if (cdate == null) logger.error("Date extraction failed on file= {} dateExtractor = {}", mfile.getPath(), dateExtractor); files.add( new DatedMFile(mfile, cdate)); } Collections.sort(files); CalendarDateFormatter cdf = new CalendarDateFormatter("yyyyMMdd"); List<CollectionManager> result = new ArrayList<CollectionManager>(); TimePartitionCollectionManager curr = null; for (DatedMFile dmf : files) { if ((curr == null) || (!curr.endPartition.isAfter(dmf.cdate))) { CalendarPeriod period = CalendarPeriod.of(config.timePartition); CalendarDate start = dmf.cdate.truncate(period.getField()); // start on a boundary CalendarDate end = start.add( period); String name = collectionName + "-"+ cdf.toString(dmf.cdate); curr = new TimePartitionCollectionManager(name, start, end, getRoot(), this.auxInfo, this.logger); result.add(curr); } curr.add(dmf); } return result; }
@Test public void testFeatureCollectionConfig() throws IOException { FeatureCollectionConfig config = new FeatureCollectionConfig("fnmoc", "test/fnmoc", FeatureCollectionType.GRIB1, "[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America/FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm#.grib1", null, null, null, "file", null); DateExtractor extractor = config.getDateExtractor(); String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }
MCollection mcs = new MCollection(mfile.getName(), mfile.getPath(), sp.wantSubdirs(), filters, null); CalendarDate cdate = dateExtractor.getCalendarDate(mfile); if (cdate == null) { logger.error("TimePartitionCollections dateExtractor = {} not working on mfile = {} ", dateExtractor, mfile.getPath());
public Iterable<MCollection> makePartitions(CollectionUpdateType forceCollection) throws IOException { List<MCollection> result = new ArrayList<>(); CollectionListRange curr = null; CalendarDate startDate = null; CalendarDate endDate = null; for (MFile mfile : getFilesSorted()) { CalendarDate cdate = dateExtractor.getCalendarDate(mfile); if (cdate == null) continue; // skip - error should be logged if ((curr == null) || !endDate.isAfter(cdate)) { startDate = cdate.truncate(timePeriod.getField()); // start on a boundary endDate = startDate.add( timePeriod); String name = collectionName + "-"+ cdf.toString(startDate); curr = new CollectionListRange(name, root, startDate, endDate, this.logger); curr.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, this.config); if (!wasRemoved( curr)) result.add(curr); // skip if in removed list } //System.out.printf("%s%n", mfile); curr.addFile(mfile); } return result; }
@Test public void testParseCalendarDate() throws IOException { Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser("/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km/GFS_CONUS_95km_#yyyyMMdd_HHmm#.grib2", errlog); System.out.printf("errlog=%s%n", errlog); System.out.printf("specp=%s%n", specp); // test parsing assert specp.getRootDir().equals("/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km"); assert !specp.wantSubdirs(); assert specp.getFilter().toString().equals("GFS_CONUS_95km_..............grib2"); assert specp.getDateFormatMark().equals("GFS_CONUS_95km_#yyyyMMdd_HHmm"); // test filter MFileFilter mfilter = new WildcardMatchOnName(specp.getFilter()); String path = "/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km/GFS_CONUS_95km_20141203_0000.grib2"; MFile mfile = new MFileOS7(Paths.get(path), null); assert mfilter.accept(mfile); // 2014-12-03T16:43:59.433 -0700 ERROR - ucar.nc2.units.DateFromString - Must delineate Date between 2 '#' chars, dateFormatString = GFS_CONUS_95km_#yyyyMMdd_HHmm // test date extractor DateExtractor extractor = new DateExtractorFromName(specp.getDateFormatMark(), true); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-03T00:00:00Z")); }
@Test @Category(NeedsCdmUnitTest.class) public void testFeatureCollectionReader() throws IOException, JDOMException { File configFile = new File(TestDir.cdmUnitTestDir+"tds/config/fnmoc.xml"); org.jdom2.Document doc; SAXBuilder builder = new SAXBuilder(); doc = builder.build(configFile); XMLOutputter xmlOut = new XMLOutputter(); System.out.println(xmlOut.outputString(doc)); Formatter errlog = new Formatter(); FeatureCollectionConfigBuilder fcb = new FeatureCollectionConfigBuilder(errlog); FeatureCollectionConfig config = fcb.readConfig(doc.getRootElement()); DateExtractor extractor = config.getDateExtractor(); String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }