public CollectionGeneral(FeatureCollectionConfig config, CollectionSpecParser specp, Logger logger) { super(config.collectionName, logger); this.root = specp.getRootDir(); this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString( config.olderThan); }
/** * This is only used for the top level GribCollection. * * @param config use this FeatureCollectionConfig * @return index File */ static private File makeTopIndexFileFromConfig(FeatureCollectionConfig config) { Formatter errlog = new Formatter(); CollectionSpecParser specp = config.getCollectionSpecParser(errlog); String name = StringUtil2.replace(config.collectionName, '\\', "/"); // String cname = DirectoryCollection.makeCollectionName(name, Paths.get(specp.getRootDir())); return makeIndexFile(name, new File(specp.getRootDir())); }
public FeatureCollectionRef(DatasetNode parent, String name, String xlink, Map<String, Object> flds, List<AccessBuilder> accessBuilders, List<DatasetBuilder> datasetBuilders, thredds.featurecollection.FeatureCollectionConfig config) { super(parent, name, xlink, flds, accessBuilders, datasetBuilders); this.config = config; this.config.spec = AliasTranslator.translateAlias(this.config.spec); // <collection spec="${cdmUnitTest}/ncss/CONUS_80km_nc/GFS_CONUS_80km_#yyyyMMdd_HHmm#.nc$" /> CollectionSpecParser specp = new CollectionSpecParser(config.spec, null); topDirectoryLocation = specp.getRootDir(); }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
private MFileCollectionManager(String collectionSpec, String olderThan, Formatter errlog) { super(collectionSpec, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(2); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
private MFileCollectionManager(String collectionName, String collectionSpec, String olderThan, Formatter errlog) { super(collectionName, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(2); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
private MFileCollectionManager(String collectionName, String collectionSpec, String olderThan, Formatter errlog) { super(collectionName, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
MCollection mc = new MCollection(sp.getRootDir(), sp.getRootDir(), false, (MFileFilter) null, null); File root = new File(sp.getRootDir()); if (!root.exists()) { logger.error("TimePartitionCollections root = {} does not exist ", sp.getRootDir()); return null; logger.error("TimePartitionCollections Invalid collection, no subdirectories found; root = {}, collection= {} ", sp.getRootDir(), mc); return null;
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.name != null ? config.name : config.spec, logger); this.config = config; CollectionSpecParser sp = new CollectionSpecParser(config.spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); if (config.protoConfig != null) protoChoice = config.protoConfig.choice; if (config.updateConfig != null) { this.recheck = makeRecheck(config.updateConfig.recheckAfter); // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); } if (this.auxInfo == null) this.auxInfo = new HashMap<>(10); this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config); }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.name != null ? config.name : config.spec, logger); this.config = config; CollectionSpecParser sp = new CollectionSpecParser(config.spec, errlog); this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = makeRecheck(config.updateConfig.recheckAfter); protoChoice = config.protoConfig.choice; // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); }
public CollectionPathMatcher(FeatureCollectionConfig config, CollectionSpecParser specp, Logger logger) { super(config.collectionName, logger); this.config = config; this.wantSubdirs = specp.wantSubdirs(); setRoot(specp.getRootDir()); DateExtractor extract = config.getDateExtractor(); if (extract != null && !(extract instanceof DateExtractorNone)) setDateExtractor(extract); putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); matcher = specp.getPathMatcher(); // LOOK still need to decide what you are matching on name, path, etc this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString( config.olderThan); }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.collectionName != null ? config.collectionName : config.spec, logger); this.config = config; CollectionSpecParser sp = config.getCollectionSpecParser(errlog); this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); if (config.protoConfig != null) protoChoice = config.protoConfig.choice; if (config.updateConfig != null) { this.recheck = makeRecheck(config.updateConfig.recheckAfter); // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); } if (this.auxInfo == null) this.auxInfo = new HashMap<>(10); this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config); }
@Test @Ignore("tests fail on jenkins due to file permisssions") public void testOlderThanInDirectoryCollection() throws IOException { // public FeatureCollectionConfig(String name, String path, FeatureCollectionType fcType, String spec, // String dateFormatMark, String olderThan, // String timePartition, String useIndexOnlyS, Element innerNcml) { FeatureCollectionConfig config = new FeatureCollectionConfig("testOlderThanInDirectoryCollection", "path", FeatureCollectionType.GRIB1, TestDir.cdmUnitTestDir + "datasets/NDFD-CONUS-5km/.*grib2", null, null, "30 sec", null, null); Formatter errlog = new Formatter(System.out); CollectionSpecParser specp = new CollectionSpecParser(config.spec, errlog); String fileToTouch = specp.getRootDir() + "/NDFD_CONUS_5km_20131213_1200.grib2"; assert touch(fileToTouch); // count scanned files // String topCollectionName, String topDirS, String olderThan, org.slf4j.Logger logger DirectoryCollection dcm = new DirectoryCollection("topCollectionName", specp.getRootDir(), true, config.olderThan, logger); dcm.setStreamFilter(new StreamFilter(java.util.regex.Pattern.compile(".*grib2"), true)); List<String> fileList = dcm.getFilenames(); for (String name : fileList) System.out.printf("%s%n", name); assert fileList.size() == 3 : fileList.size()+" != 3 in "+specp.getRootDir(); }
Path rootPath = Paths.get(specp.getRootDir()); boolean isGrib1 = config.type == FeatureCollectionType.GRIB1;
@Test public void testCalendarDateFromPath() throws IOException { Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser("[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America/FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm#.grib1", errlog); System.out.printf("errlog=%s%n", errlog); System.out.printf("specp=%s%n", specp); // test parsing assert specp.getRootDir().equals("[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America"); assert !specp.wantSubdirs(); assert specp.getFilter().toString().equals("FNMOC_COAMPS_Equatorial_America_..............grib1"); assert specp.getDateFormatMark().equals("FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm"); // test filter String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); // test date extractor DateExtractor extractor = new DateExtractorFromName(specp.getDateFormatMark(), true); CalendarDate cd = extractor.getCalendarDateFromPath(mfile.toString()); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }
@Test public void testParseCalendarDate() throws IOException { Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser("/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km/GFS_CONUS_95km_#yyyyMMdd_HHmm#.grib2", errlog); System.out.printf("errlog=%s%n", errlog); System.out.printf("specp=%s%n", specp); // test parsing assert specp.getRootDir().equals("/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km"); assert !specp.wantSubdirs(); assert specp.getFilter().toString().equals("GFS_CONUS_95km_..............grib2"); assert specp.getDateFormatMark().equals("GFS_CONUS_95km_#yyyyMMdd_HHmm"); // test filter MFileFilter mfilter = new WildcardMatchOnName(specp.getFilter()); String path = "/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km/GFS_CONUS_95km_20141203_0000.grib2"; MFile mfile = new MFileOS7(Paths.get(path), null); assert mfilter.accept(mfile); // 2014-12-03T16:43:59.433 -0700 ERROR - ucar.nc2.units.DateFromString - Must delineate Date between 2 '#' chars, dateFormatString = GFS_CONUS_95km_#yyyyMMdd_HHmm // test date extractor DateExtractor extractor = new DateExtractorFromName(specp.getDateFormatMark(), true); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-03T00:00:00Z")); }
@Test public void testStreamFilterInDirPartition() throws IOException { // this dataset 0-6 hour forecasts x 124 runtimes (4x31) // there are 2 groups, likely miscoded, the smaller group are 0 hour, duplicates, possibly miscoded FeatureCollectionConfig config = new FeatureCollectionConfig("cfrsAnalysis_46", "test/testCfrsAnalysisOnly", FeatureCollectionType.GRIB2, TestDir.cdmUnitTestDir + "gribCollections/cfsr/.*grb2", null, null, null, "directory", null); Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser(config.spec, errlog); Path rootPath = Paths.get(specp.getRootDir()); try (DirectoryCollection dcm = new DirectoryCollection(config.collectionName, rootPath, true, config.olderThan, logger)) { dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName())); int count = 0; for (MFile mfile : dcm.getFilesSorted()) { System.out.printf("%s%n", mfile); assert mfile.getName().equals("pwat.gdas.199612.grb2"); count++; } assert count == 1; } }