@Override public Void call() throws Exception { SerializationUtils.serializeState(ParallelRunner.this.fs, outputFilePath, state); return null; } }), "Serialize state to " + outputFilePath));
/** * Serialize a {@link State} instance to a file. * * @param fs the {@link FileSystem} instance for creating the file * @param jobStateFilePath the path to the file * @param state the {@link State} to serialize * @param <T> the {@link State} object type * @throws IOException if it fails to serialize the {@link State} instance */ public static <T extends State> void serializeState(FileSystem fs, Path jobStateFilePath, T state) throws IOException { serializeState(fs, jobStateFilePath, state, fs.getDefaultReplication(jobStateFilePath)); }
/** * Create a job from a given batch of {@link WorkUnit}s. */ private JobConfig.Builder createJob(List<WorkUnit> workUnits) throws IOException { Map<String, TaskConfig> taskConfigMap = Maps.newHashMap(); try (ParallelRunner stateSerDeRunner = new ParallelRunner(this.stateSerDeRunnerThreads, this.fs)) { int multiTaskIdSequence = 0; for (WorkUnit workUnit : workUnits) { if (workUnit instanceof MultiWorkUnit) { workUnit.setId(JobLauncherUtils.newMultiTaskId(this.jobContext.getJobId(), multiTaskIdSequence++)); } addWorkUnit(workUnit, stateSerDeRunner, taskConfigMap); } Path jobStateFilePath = new Path(this.appWorkDir, this.jobContext.getJobId() + "." + JOB_STATE_FILE_NAME); SerializationUtils.serializeState(this.fs, jobStateFilePath, this.jobContext.getJobState()); } JobConfig.Builder jobConfigBuilder = new JobConfig.Builder(); jobConfigBuilder.setMaxAttemptsPerTask(this.jobContext.getJobState().getPropAsInt( ConfigurationKeys.MAX_TASK_RETRIES_KEY, ConfigurationKeys.DEFAULT_MAX_TASK_RETRIES)); jobConfigBuilder.setFailureThreshold(workUnits.size()); jobConfigBuilder.addTaskConfigMap(taskConfigMap).setCommand(GobblinTaskRunner.GOBBLIN_TASK_FACTORY_NAME); jobConfigBuilder.setNumConcurrentTasksPerInstance(ConfigUtils.getInt(jobConfig, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY_DEFAULT)); return jobConfigBuilder; }