/** * Create a new empty {@link MultiWorkUnit} instance. * * @return a new empty {@link MultiWorkUnit} instance */ public static MultiWorkUnit createEmpty() { return new MultiWorkUnit(); } }
private static long getMultiWorkUnitWeight(MultiWorkUnit multiWorkUnit) { return multiWorkUnit.contains(TOTAL_MULTI_WORK_UNIT_WEIGHT) ? multiWorkUnit.getPropAsLong(TOTAL_MULTI_WORK_UNIT_WEIGHT) : 0; }
private static void setMultiWorkUnitWeight(MultiWorkUnit multiWorkUnit, long weight) { multiWorkUnit.setProp(TOTAL_MULTI_WORK_UNIT_WEIGHT, Long.toString(weight)); }
@Override public WorkUnit next() { if (this.currentIterator != null && this.currentIterator.hasNext()) { WorkUnit next = this.currentIterator.next(); if (next instanceof MultiWorkUnit) { throw new IllegalStateException("A MultiWorkUnit cannot contain other MultiWorkUnits."); } return next; } WorkUnit wu = this.workUnits.next(); if (wu instanceof MultiWorkUnit) { this.currentIterator = ((MultiWorkUnit) wu).getWorkUnits().iterator(); return next(); } else { return wu; } }
MultiWorkUnit mwu = MultiWorkUnit.createEmpty(); try { mwu.readFields(workUnitFileCloser.register(new DataInputStream(fs.open(status.getPath())))); } finally { workUnitFileCloser.close(); for (WorkUnit wu : mwu.getWorkUnits()) { JobLauncherUtils.cleanTaskStagingData(new WorkUnitState(wu), LOG);
@Override public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { WorkUnit workUnit = (value.toString().endsWith(MULTI_WORK_UNIT_FILE_EXTENSION) ? MultiWorkUnit.createEmpty() : WorkUnit.createEmpty()); SerializationUtils.deserializeState(this.fs, new Path(value.toString()), workUnit); if (workUnit instanceof MultiWorkUnit) { List<WorkUnit> flattenedWorkUnits = JobLauncherUtils.flattenWorkUnits(((MultiWorkUnit) workUnit).getWorkUnits()); this.workUnits.addAll(flattenedWorkUnits); } else { this.workUnits.add(workUnit); } } }
pQueue.add(MultiWorkUnit.createEmpty()); } else { MultiWorkUnit newMultiWorkUnit = MultiWorkUnit.createEmpty(); addToMultiWorkUnit(newMultiWorkUnit, workUnit, weight); pQueue.add(newMultiWorkUnit);
private static void addToMultiWorkUnit(MultiWorkUnit multiWorkUnit, WorkUnit workUnit, long weight) { multiWorkUnit.addWorkUnit(workUnit); setMultiWorkUnitWeight(multiWorkUnit, getMultiWorkUnitWeight(multiWorkUnit) + weight); }
/** * Utility method that takes in a {@link List} of {@link WorkUnit}s, and flattens them. It builds up * the flattened list by checking each element of the given list, and seeing if it is an instance of * {@link MultiWorkUnit}. If it is then it calls itself on the {@link WorkUnit}s returned by * {@link MultiWorkUnit#getWorkUnits()}. If not, then it simply adds the {@link WorkUnit} to the * flattened list. * * @param workUnits is a {@link List} containing either {@link WorkUnit}s or {@link MultiWorkUnit}s * @return a {@link List} of flattened {@link WorkUnit}s */ public static List<WorkUnit> flattenWorkUnits(Collection<WorkUnit> workUnits) { List<WorkUnit> flattenedWorkUnits = Lists.newArrayList(); for (WorkUnit workUnit : workUnits) { if (workUnit instanceof MultiWorkUnit) { flattenedWorkUnits.addAll(flattenWorkUnits(((MultiWorkUnit) workUnit).getWorkUnits())); } else { flattenedWorkUnits.add(workUnit); } } return flattenedWorkUnits; }
@Nullable @Override public WorkUnit apply(WorkUnit input) { if (input instanceof MultiWorkUnit) { for (WorkUnit wu : ((MultiWorkUnit) input).getWorkUnits()) { forWorkUnit(wu); } } else { forWorkUnit(input); } return input; }
JobLauncherUtils.flattenWorkUnits(((MultiWorkUnit) workUnit).getWorkUnits()); workUnits.addAll(flattenedWorkUnits); } else {
@Override public boolean apply(WorkUnit workUnit) { if (workUnit instanceof MultiWorkUnit) { Preconditions.checkArgument(!workUnit.contains(ConfigurationKeys.WORK_UNIT_SKIP_KEY), "Error: MultiWorkUnit cannot be skipped"); for (WorkUnit wu : ((MultiWorkUnit) workUnit).getWorkUnits()) { Preconditions.checkArgument(!wu.contains(ConfigurationKeys.WORK_UNIT_SKIP_KEY), "Error: MultiWorkUnit cannot contain skipped WorkUnit"); } } if (workUnit.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_SKIP_KEY, false)) { WorkUnitState workUnitState = new WorkUnitState(workUnit, this.jobState); workUnitState.setWorkingState(WorkUnitState.WorkingState.SKIPPED); this.jobState.addSkippedTaskState(new TaskState(workUnitState)); return false; } return true; } }