public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
public static void main(String arg[]) { doit("/data/ldm/pub/native/grid/NCEP/GFS/Alaska_191km/**/GFS_Alaska_191km_#yyyyMMdd_HHmm#\\.grib1$", new Formatter()); doit("Q:/grid/grib/grib1/data/agg/.*\\.grb", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm#\\.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm#.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_#yyyyMMdd_HHmm", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_#yyyyMMdd_HHmm#.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_yyyyMMdd_HHmm.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/*", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/*", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/T*.T", new Formatter()); }
public FeatureCollectionRef(DatasetNode parent, String name, String xlink, Map<String, Object> flds, List<AccessBuilder> accessBuilders, List<DatasetBuilder> datasetBuilders, thredds.featurecollection.FeatureCollectionConfig config) { super(parent, name, xlink, flds, accessBuilders, datasetBuilders); this.config = config; this.config.spec = AliasTranslator.translateAlias(this.config.spec); // <collection spec="${cdmUnitTest}/ncss/CONUS_80km_nc/GFS_CONUS_80km_#yyyyMMdd_HHmm#.nc$" /> CollectionSpecParser specp = new CollectionSpecParser(config.spec, null); topDirectoryLocation = specp.getRootDir(); }
@Test public void testStreamFilterInDirPartition() throws IOException { // this dataset 0-6 hour forecasts x 124 runtimes (4x31) // there are 2 groups, likely miscoded, the smaller group are 0 hour, duplicates, possibly miscoded FeatureCollectionConfig config = new FeatureCollectionConfig("cfrsAnalysis_46", "test/testCfrsAnalysisOnly", FeatureCollectionType.GRIB2, TestDir.cdmUnitTestDir + "gribCollections/cfsr/.*grb2", null, null, null, "directory", null); Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser(config.spec, errlog); Path rootPath = Paths.get(specp.getRootDir()); try (DirectoryCollection dcm = new DirectoryCollection(config.collectionName, rootPath, true, config.olderThan, logger)) { dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName())); int count = 0; for (MFile mfile : dcm.getFilesSorted()) { System.out.printf("%s%n", mfile); assert mfile.getName().equals("pwat.gdas.199612.grb2"); count++; } assert count == 1; } }
public CollectionPathMatcher(FeatureCollectionConfig config, CollectionSpecParser specp, Logger logger) { super(config.collectionName, logger); this.config = config; this.wantSubdirs = specp.wantSubdirs(); setRoot(specp.getRootDir()); DateExtractor extract = config.getDateExtractor(); if (extract != null && !(extract instanceof DateExtractorNone)) setDateExtractor(extract); putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); matcher = specp.getPathMatcher(); // LOOK still need to decide what you are matching on name, path, etc this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString( config.olderThan); }
Path rootPath = Paths.get(specp.getRootDir()); boolean isGrib1 = config.type == FeatureCollectionType.GRIB1; if (specp.wantSubdirs()) { // its a partition
public CollectionSpecParser getCollectionSpecParser(Formatter errlog) { if (rootDir != null) return new CollectionSpecParser(rootDir, regExp, errlog); else return new CollectionSpecParser(spec, errlog); }
/** * Update all the gbx indices in one directory, and the ncx index for that directory * * @param config FeatureCollectionConfig * @param dirPath directory path * @return true if collection was rewritten, exception on failure * @throws IOException */ static private boolean updateLeafCollection(boolean isGrib1, FeatureCollectionConfig config, CollectionUpdateType updateType, boolean isTop, Logger logger, Path dirPath) throws IOException { if (config.ptype == FeatureCollectionConfig.PartitionType.file) { return updateFilePartition(isGrib1, config, updateType, isTop, logger, dirPath); } else { Formatter errlog = new Formatter(); CollectionSpecParser specp = config.getCollectionSpecParser(errlog); try (DirectoryCollection dcm = new DirectoryCollection(config.collectionName, dirPath, isTop, config.olderThan, logger)) { dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName())); boolean changed = updateGribCollection(isGrib1, dcm, updateType, FeatureCollectionConfig.PartitionType.directory, logger, errlog); if (debug) System.out.printf(" GribCdmIndex.updateDirectoryPartition was updated=%s on %s%n", changed, dirPath); return changed; } } }
public CollectionGeneral(FeatureCollectionConfig config, CollectionSpecParser specp, Logger logger) { super(config.collectionName, logger); this.root = specp.getRootDir(); this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString( config.olderThan); }
public DateExtractor getDateExtractor() { if (dateFormatMark != null) return new DateExtractorFromName(dateFormatMark, false); else { CollectionSpecParser sp = getCollectionSpecParser(null); if (sp.getDateFormatMark() != null) return new DateExtractorFromName(sp.getDateFormatMark(), true); } return new DateExtractorNone(); }
@Test @Ignore("tests fail on jenkins due to file permisssions") public void testOlderThanInDirectoryCollection() throws IOException { // public FeatureCollectionConfig(String name, String path, FeatureCollectionType fcType, String spec, // String dateFormatMark, String olderThan, // String timePartition, String useIndexOnlyS, Element innerNcml) { FeatureCollectionConfig config = new FeatureCollectionConfig("testOlderThanInDirectoryCollection", "path", FeatureCollectionType.GRIB1, TestDir.cdmUnitTestDir + "datasets/NDFD-CONUS-5km/.*grib2", null, null, "30 sec", null, null); Formatter errlog = new Formatter(System.out); CollectionSpecParser specp = new CollectionSpecParser(config.spec, errlog); String fileToTouch = specp.getRootDir() + "/NDFD_CONUS_5km_20131213_1200.grib2"; assert touch(fileToTouch); // count scanned files // String topCollectionName, String topDirS, String olderThan, org.slf4j.Logger logger DirectoryCollection dcm = new DirectoryCollection("topCollectionName", specp.getRootDir(), true, config.olderThan, logger); dcm.setStreamFilter(new StreamFilter(java.util.regex.Pattern.compile(".*grib2"), true)); List<String> fileList = dcm.getFilenames(); for (String name : fileList) System.out.printf("%s%n", name); assert fileList.size() == 3 : fileList.size()+" != 3 in "+specp.getRootDir(); }
private static void doit(String spec, Formatter errlog) { CollectionSpecParser specp = new CollectionSpecParser(spec, errlog); System.out.printf("spec= %s%n%s%n", spec, specp); String err = errlog.toString(); if (err.length() > 0) System.out.printf("%s%n", err); System.out.printf("-----------------------------------%n"); }
if (specp.getFilter() != null) partition.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));
/** * This is only used for the top level GribCollection. * * @param config use this FeatureCollectionConfig * @return index File */ static private File makeTopIndexFileFromConfig(FeatureCollectionConfig config) { Formatter errlog = new Formatter(); CollectionSpecParser specp = config.getCollectionSpecParser(errlog); String name = StringUtil2.replace(config.collectionName, '\\', "/"); // String cname = DirectoryCollection.makeCollectionName(name, Paths.get(specp.getRootDir())); return makeIndexFile(name, new File(specp.getRootDir())); }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
private static void doit(String spec, Formatter errlog) { CollectionSpecParser specp = new CollectionSpecParser(spec, errlog); System.out.printf("spec= %s%n%s%n", spec, specp); String err = errlog.toString(); if (err.length() > 0) System.out.printf("%s%n", err); System.out.printf("-----------------------------------%n"); }
public static void main(String arg[]) { doit("/data/ldm/pub/native/grid/NCEP/GFS/Alaska_191km/**/GFS_Alaska_191km_#yyyyMMdd_HHmm#\\.grib1$", new Formatter()); doit("Q:/grid/grib/grib1/data/agg/.*\\.grb", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm#\\.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm#.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_#yyyyMMdd_HHmm", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_#yyyyMMdd_HHmm#.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_yyyyMMdd_HHmm.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/*", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/*", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/T*.T", new Formatter()); }
private MFileCollectionManager(String collectionSpec, String olderThan, Formatter errlog) { super(collectionSpec, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(2); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
private static void doit(String spec, Formatter errlog) { CollectionSpecParser specp = new CollectionSpecParser(spec, errlog); System.out.printf("spec= %s%n%s%n", spec, specp); String err = errlog.toString(); if (err.length() > 0) System.out.printf("%s%n", err); System.out.printf("-----------------------------------%n"); }
public static void main(String arg[]) { doit("/u00/FNMOC/NAVGEM/pressure/**/US058GMET-GR1mdl.0018_0056_00000F0..#yyyyMMddHH#_0102_000000-000000pres$", new Formatter()); doit("/data/ldm/pub/native/grid/NCEP/GFS/Alaska_191km/**/GFS_Alaska_191km_#yyyyMMdd_HHmm#\\.grib1$", new Formatter()); doit("Q:/grid/grib/grib1/data/agg/.*\\.grb", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm#\\.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm#.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/Surface_METAR_#yyyyMMdd_HHmm", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_#yyyyMMdd_HHmm", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_#yyyyMMdd_HHmm#.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/Surface_METAR_yyyyMMdd_HHmm.nc", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/**/*", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/*", new Formatter()); doit("/data/ldm/pub/decoded/netcdf/surface/metar/T*.T", new Formatter()); }