@Override public Void call() throws Exception { SerializationUtils.deserializeState(ParallelRunner.this.fs, inputFilePath, state); return null; } }), "Deserialize state from " + inputFilePath));
private T deserializeMetadata(byte[] bytes) throws IOException, ClassNotFoundException { return (T) SerializationUtils.deserializeFromBytes(bytes, Metadata.class); }
/** * Deserialize/read a {@link State} instance from a file. * * @param fs the {@link FileSystem} instance for opening the file * @param jobStateFilePath the path to the file * @param state an empty {@link State} instance to deserialize into * @param <T> the {@link State} object type * @throws IOException if it fails to deserialize the {@link State} instance */ public static <T extends State> void deserializeState(FileSystem fs, Path jobStateFilePath, T state) throws IOException { try (InputStream is = fs.open(jobStateFilePath)) { deserializeStateFromInputStream(is, state); } }
@Override public Void call() throws Exception { SerializationUtils.serializeState(ParallelRunner.this.fs, outputFilePath, state); return null; } }), "Serialize state to " + outputFilePath));
private byte[] serializeMetadata(T metadata) throws IOException { return SerializationUtils.serializeIntoBytes(metadata); }
/** * Deserialize a String obtained via {@link #serialize(Serializable)} into an object, using * {@link BaseEncoding#base64()}. * * @param serialized The serialized String * @param clazz The class the deserialized object should be cast to. * @return The deserialized object * @throws IOException if it fails to deserialize the object */ public static <T extends Serializable> T deserialize(String serialized, Class<T> clazz) throws IOException { return deserialize(serialized, clazz, DEFAULT_ENCODING); }
/** * Serialize an object into a String. The object is first serialized into a byte array, * which is converted into a String using {@link BaseEncoding#base64()}. * * @param obj A {@link Serializable} object * @return A String representing the input object * @throws IOException if it fails to serialize the object */ public static <T extends Serializable> String serialize(T obj) throws IOException { return serialize(obj, DEFAULT_ENCODING); }
/** * Serialize a {@link State} instance to a file. * * @param fs the {@link FileSystem} instance for creating the file * @param jobStateFilePath the path to the file * @param state the {@link State} to serialize * @param <T> the {@link State} object type * @throws IOException if it fails to serialize the {@link State} instance */ public static <T extends State> void serializeState(FileSystem fs, Path jobStateFilePath, T state) throws IOException { serializeState(fs, jobStateFilePath, state, fs.getDefaultReplication(jobStateFilePath)); }
/** * Serialize an object into a String. The object is first serialized into a byte array, * which is converted into a String using the given {@link BaseEncoding}. * * @param obj A {@link Serializable} object * @param enc The {@link BaseEncoding} used to encode a byte array. * @return A String representing the input object * @throws IOException if it fails to serialize the object */ public static <T extends Serializable> String serialize(T obj, BaseEncoding enc) throws IOException { return enc.encode(serializeIntoBytes(obj)); }
@Override public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { WorkUnit workUnit = (value.toString().endsWith(MULTI_WORK_UNIT_FILE_EXTENSION) ? MultiWorkUnit.createEmpty() : WorkUnit.createEmpty()); SerializationUtils.deserializeState(this.fs, new Path(value.toString()), workUnit); if (workUnit instanceof MultiWorkUnit) { List<WorkUnit> flattenedWorkUnits = JobLauncherUtils.flattenWorkUnits(((MultiWorkUnit) workUnit).getWorkUnits()); this.workUnits.addAll(flattenedWorkUnits); } else { this.workUnits.add(workUnit); } } }
/** * Create a job from a given batch of {@link WorkUnit}s. */ private JobConfig.Builder createJob(List<WorkUnit> workUnits) throws IOException { Map<String, TaskConfig> taskConfigMap = Maps.newHashMap(); try (ParallelRunner stateSerDeRunner = new ParallelRunner(this.stateSerDeRunnerThreads, this.fs)) { int multiTaskIdSequence = 0; for (WorkUnit workUnit : workUnits) { if (workUnit instanceof MultiWorkUnit) { workUnit.setId(JobLauncherUtils.newMultiTaskId(this.jobContext.getJobId(), multiTaskIdSequence++)); } addWorkUnit(workUnit, stateSerDeRunner, taskConfigMap); } Path jobStateFilePath = new Path(this.appWorkDir, this.jobContext.getJobId() + "." + JOB_STATE_FILE_NAME); SerializationUtils.serializeState(this.fs, jobStateFilePath, this.jobContext.getJobState()); } JobConfig.Builder jobConfigBuilder = new JobConfig.Builder(); jobConfigBuilder.setMaxAttemptsPerTask(this.jobContext.getJobState().getPropAsInt( ConfigurationKeys.MAX_TASK_RETRIES_KEY, ConfigurationKeys.DEFAULT_MAX_TASK_RETRIES)); jobConfigBuilder.setFailureThreshold(workUnits.size()); jobConfigBuilder.addTaskConfigMap(taskConfigMap).setCommand(GobblinTaskRunner.GOBBLIN_TASK_FACTORY_NAME); jobConfigBuilder.setNumConcurrentTasksPerInstance(ConfigUtils.getInt(jobConfig, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY_DEFAULT)); return jobConfigBuilder; }
/** * Deserialize a String obtained via {@link #serialize(Serializable)} into an object, using the * given {@link BaseEncoding}, which must be the same {@link BaseEncoding} used to serialize the object. * * @param serialized The serialized String * @param clazz The class the deserialized object should be cast to. * @param enc The {@link BaseEncoding} used to decode the String. * @return The deserialized object * @throws IOException if it fails to deserialize the object */ public static <T extends Serializable> T deserialize(String serialized, Class<T> clazz, BaseEncoding enc) throws IOException { return deserializeFromBytes(enc.decode(serialized), clazz); }
if (dcPath.getName().equals(jobStateFileName)) { SerializationUtils .deserializeStateFromInputStream(closer.register(new FileInputStream(dcPath.toUri().getPath())), this.jobState); foundStateFile = true;
public GobblinHelixTask(TaskCallbackContext taskCallbackContext, Optional<ContainerMetrics> containerMetrics, TaskExecutor taskExecutor, TaskStateTracker taskStateTracker, FileSystem fs, Path appWorkDir, StateStores stateStores) throws IOException { this.taskExecutor = taskExecutor; this.taskStateTracker = taskStateTracker; this.taskConfig = taskCallbackContext.getTaskConfig(); this.jobName = this.taskConfig.getConfigMap().get(ConfigurationKeys.JOB_NAME_KEY); this.jobId = this.taskConfig.getConfigMap().get(ConfigurationKeys.JOB_ID_KEY); this.jobKey = Long.toString(Id.parse(this.jobId).getSequence()); this.participantId = taskCallbackContext.getManager().getInstanceName(); this.fs = fs; this.stateStores = stateStores; Path jobStateFilePath = new Path(appWorkDir, this.jobId + "." + AbstractJobLauncher.JOB_STATE_FILE_NAME); SerializationUtils.deserializeState(this.fs, jobStateFilePath, this.jobState); if (containerMetrics.isPresent()) { // This must be done after the jobState is deserialized from the jobStateFilePath // A reference to jobMetrics is required to ensure it is not evicted from the GobblinMetricsRegistry Cache this.jobMetrics = Optional.of(JobMetrics.get(this.jobState, containerMetrics.get().getMetricContext())); } else { this.jobMetrics = Optional.absent(); } }