/** * Materialize a {@link WorkUnitStream} into an in-memory list. Note that infinite work unit streams cannot be materialized. */ private List<WorkUnit> materializeWorkUnitList(WorkUnitStream workUnitStream) { if (!workUnitStream.isFiniteStream()) { throw new UnsupportedOperationException("Cannot materialize an infinite work unit stream."); } return Lists.newArrayList(workUnitStream.getWorkUnits()); }
/** * Prepare the flattened {@link WorkUnit}s for execution by populating the job and task IDs. */ private WorkUnitStream prepareWorkUnits(WorkUnitStream workUnits, JobState jobState) { return workUnits.transform(new WorkUnitPreparator(this.jobContext.getJobId())); }
if (workUnitStream == null || workUnitStream.getWorkUnits() == null) { this.eventSubmitter.submit(JobEvent.WORK_UNITS_MISSING); jobState.setState(JobState.RunningState.FAILED); if (!workUnitStream.getWorkUnits().hasNext()) { this.eventSubmitter.submit(JobEvent.WORK_UNITS_EMPTY); LOG.warn("No work units have been created for job " + jobId); workUnitStream = workUnitStream.filter(new SkippedWorkUnitsFilter(jobState)); workUnitStream = workUnitStream.transform(new MultiWorkUnitForEach() { @Override public void forWorkUnit(WorkUnit workUnit) {
if (workUnits.isSafeToMaterialize()) { Closer closer = Closer.create(); Map<String, ParallelRunner> parallelRunners = Maps.newHashMap(); try { for (WorkUnit workUnit : JobLauncherUtils.flattenWorkUnits(workUnits.getMaterializedWorkUnitCollection())) { JobLauncherUtils.cleanTaskStagingData(new WorkUnitState(workUnit, jobState), LOG, closer, parallelRunners);
final JobState jobState = this.jobContext.getJobState(); Iterator<WorkUnit> workUnitIterator = workUnitStream.getWorkUnits(); if (!workUnitIterator.hasNext()) { LOG.warn("No work units to run"); Iterator<WorkUnit> flattenedWorkUnits = new MultiWorkUnitUnpackingIterator(workUnitStream.getWorkUnits()); Iterator<WorkUnit> workUnitsWithJobState = Iterators.transform(flattenedWorkUnits, new Function<WorkUnit, WorkUnit>() { @Override