/** * Determine the date range for inputs to process based on the configuration and available inputs. */ protected void determineDateRange() { _log.info("Determining range of input data to consume"); _range = DateRangePlanner.getDateRange(getStartDate(), getEndDate(), getAvailableInputsByDate().keySet(), getDaysAgo(), getNumDays(), isFailOnMissing()); if (_range.getBeginDate() == null || _range.getEndDate() == null) { throw new RuntimeException("Expected start and end date"); } } }
/** * Determine what input data is available. * * @throws IOException IOException */ protected void loadInputData() throws IOException { _inputPathsByDate = new ArrayList<SortedMap<Date,DatePath>>(); for (Path inputPath : getInputPaths()) { _log.info(String.format("Searching for available input data in " + inputPath)); _inputPathsByDate.add(getDailyData(inputPath)); } }
/** * Get a map from date to path for all paths matching yyyy/MM/dd under the given path. * * @param path path to search under * @return map of date to path * @throws IOException IOException */ protected SortedMap<Date,DatePath> getDailyData(Path path) throws IOException { SortedMap<Date,DatePath> data = new TreeMap<Date,DatePath>(); for (DatePath datePath : PathUtils.findNestedDatedPaths(getFileSystem(),path)) { data.put(datePath.getDate(),datePath); } return data; }
Date currentDate = dates.peek(); int found = 0; int needed = getInputPaths().size(); while (currentDate != null)
/** * Get a map from date to path for all paths matching yyyyMMdd under the given path. * * @param path path to search under * @return map of date to path * @throws IOException IOException */ protected SortedMap<Date,DatePath> getDatedData(Path path) throws IOException { SortedMap<Date,DatePath> data = new TreeMap<Date,DatePath>(); for (DatePath datePath : PathUtils.findDatedPaths(getFileSystem(),path)) { data.put(datePath.getDate(),datePath); } return data; }