@Override public void initialize(BatchRuntimeContext context) throws Exception { super.initialize(context); }
@Override public void run(DatasetContext datasetContext) throws Exception { SparkBatchSinkContext sinkContext = new SparkBatchSinkContext(sinkFactory, sec, datasetContext, pipelineRuntime, stageSpec); batchSink.onRunFinish(true, sinkContext); } });
@Override public void run(DatasetContext datasetContext) throws Exception { SparkBatchSinkContext sinkContext = new SparkBatchSinkContext(sinkFactory, sec, datasetContext, pipelineRuntime, stageSpec); batchSink.prepareRun(sinkContext); } });
@Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { super.configurePipeline(pipelineConfigurer); if (!config.containsMacro("tableName")) { pipelineConfigurer.createDataset(config.tableName, Table.class); } }
@Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { super.configurePipeline(pipelineConfigurer); pipelineConfigurer.createDataset(config.tableName, Table.class); if (!config.containsMacro("runtimeDatasetName")) { pipelineConfigurer.createDataset(config.runtimeDatasetName, KeyValueTable.class.getName(), DatasetProperties.EMPTY); } }
@Override public void initialize(BatchRuntimeContext context) throws Exception { super.initialize(context); }
@Override public void run(DatasetContext datasetContext) throws Exception { SparkBatchSinkContext sinkContext = new SparkBatchSinkContext(sinkFactory, sec, datasetContext, pipelineRuntime, stageSpec); batchSink.onRunFinish(false, sinkContext); } });
@Override public Iterable<Tuple2<OUT_KEY, OUT_VAL>> call(IN input) throws Exception { if (transform == null) { BatchSink<IN, OUT_KEY, OUT_VAL> batchSink = pluginFunctionContext.createPlugin(); batchSink.initialize(pluginFunctionContext.createBatchRuntimeContext()); transform = new TrackedTransform<>(batchSink, pluginFunctionContext.createStageMetrics(), pluginFunctionContext.getDataTracer(), pluginFunctionContext.getStageStatisticsCollector()); emitter = new TransformingEmitter<>(new Function<KeyValue<OUT_KEY, OUT_VAL>, Tuple2<OUT_KEY, OUT_VAL>>() { @Override public Tuple2<OUT_KEY, OUT_VAL> apply(KeyValue<OUT_KEY, OUT_VAL> input) { return new Tuple2<>(input.getKey(), input.getValue()); } }); } emitter.reset(); transform.transform(input, emitter); return emitter.getEntries(); } }
@Override public void onRunFinish(boolean succeeded, BatchSinkContext context) { super.onRunFinish(succeeded, context); if (succeeded) { try { snapshotFileSet.onSuccess(context.getLogicalStartTime()); } catch (Exception e) { LOG.error("Exception updating state file with value of latest snapshot, ", e); } try { if (config.getCleanPartitionsOlderThan() != null) { long cutoffTime = context.getLogicalStartTime() - TimeParser.parseDuration(config.getCleanPartitionsOlderThan()); snapshotFileSet.deleteMatchingPartitionsByTime(cutoffTime); LOG.debug("Cleaned up snapshots older than {}", config.getCleanPartitionsOlderThan()); } } catch (IOException e) { LOG.error("Exception occurred while cleaning up older snapshots", e); } } }