public static List<Pattern> getPatternList(State state, String propKey) { return getPatternList(state, propKey, StringUtils.EMPTY); }
private void verifyDataCompleteness() { List<Pattern> blacklist = DatasetFilterUtils.getPatternList(this.state, COMPACTION_COMPLETENESS_VERIFICATION_BLACKLIST); List<Pattern> whitelist = DatasetFilterUtils.getPatternList(this.state, COMPACTION_COMPLETENESS_VERIFICATION_WHITELIST); int numDatasetsVerifiedTogether = getNumDatasetsVerifiedTogether(); List<Dataset> datasetsToBeVerified = Lists.newArrayList(); for (Dataset dataset : this.datasets) { if (dataset.state() != UNVERIFIED) { continue; } if (shouldVerifyCompletenessForDataset(dataset, blacklist, whitelist)) { datasetsToBeVerified.add(dataset); if (datasetsToBeVerified.size() >= numDatasetsVerifiedTogether) { ListenableFuture<Results> future = this.verifier.get().verify(datasetsToBeVerified); addCallback(datasetsToBeVerified, future); datasetsToBeVerified = Lists.newArrayList(); } } else { dataset.setState(VERIFIED); } } if (!datasetsToBeVerified.isEmpty()) { ListenableFuture<Results> future = this.verifier.get().verify(datasetsToBeVerified); addCallback(datasetsToBeVerified, future); } }
@VisibleForTesting DatasetsFinder(State state, FileSystem fs) { this.state = state; this.conf = HadoopUtils.getConfFromState(state); this.fs = fs; this.inputDir = getInputDir(); this.destDir = getDestDir(); this.tmpOutputDir = getTmpOutputDir(); this.blacklist = DatasetFilterUtils.getPatternList(state, MRCompactor.COMPACTION_BLACKLIST); this.whitelist = DatasetFilterUtils.getPatternList(state, MRCompactor.COMPACTION_WHITELIST); setTopicsFromConfigStore(state); this.highPriority = getHighPriorityPatterns(); this.normalPriority = getNormalPriorityPatterns(); this.recompactDatasets = getRecompactDatasets(); }