@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { if (this.source instanceof WorkUnitStreamSource) { return ((WorkUnitStreamSource) this.source).getWorkunitStream(state); } List<WorkUnit> workUnits = this.source.getWorkunits(state); if (workUnits == null) { // Return an empty list if no work units are returned by the source workUnits = Collections.emptyList(); } return new BasicWorkUnitStream.Builder(workUnits).build(); } catch (Throwable t) { this.logger.error("Failed to get work units for job " + this.jobId, t); // Return null in case of errors return null; } }
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { fs = getSourceFileSystem(state); suite = CompactionSuiteUtils.getCompactionSuiteFactory(state).createSuite(state); initRequestAllocator(state); initJobDir(state); copyJarDependencies(state); DatasetsFinder finder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), getSourceFileSystem(state), DefaultFileSystemGlobFinder.class.getName()); List<Dataset> datasets = finder.findDatasets(); CompactionWorkUnitIterator workUnitIterator = new CompactionWorkUnitIterator (); // Spawn a single thread to create work units new Thread(new SingleWorkUnitGeneratorService (state, prioritize(datasets, state), workUnitIterator), "SingleWorkUnitGeneratorService").start(); return new BasicWorkUnitStream.Builder (workUnitIterator).build(); } catch (IOException e) { throw new RuntimeException(e); } }
workUnitStream = ((WorkUnitStreamSource) source).getWorkunitStream(jobState); } else { workUnitStream = new BasicWorkUnitStream.Builder(source.getWorkunits(jobState)).build();