public void openFilters(List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources) { for (int i = 0; i < scatterCount; i++) { TransactionalPageOutput tran = trans[i]; if (tran != null) { PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, trans[i]); filtereds[i] = filtered; closeThese[i].closeThis(filtered); } } }
public void openOutputs(OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource) { for (int i = 0; i < scatterCount; i++) { int outputTaskIndex = taskIndex * scatterCount + i; if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) { TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput( outputPlugin.open(outputTaskSource, outputSchema, outputTaskIndex)); trans[i] = tran; closeThese[i].closeThis(tran); } } }
@Override public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) { final RunnerTask task = taskSource.loadTask(RunnerTask.class); List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task); FormatterPlugin formatterPlugin = newFormatterPlugin(task); try (AbortTransactionResource aborter = new AbortTransactionResource()) { try (CloseResource closer = new CloseResource()) { TransactionalFileOutput finalOutput = PluginWrappers.transactionalFileOutput( fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex)); aborter.abortThis(finalOutput); closer.closeThis(finalOutput); FileOutput encodedOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), finalOutput); closer.closeThis(encodedOutput); PageOutput output = formatterPlugin.open(task.getFormatterTaskSource(), schema, encodedOutput); closer.closeThis(output); TransactionalPageOutput ret = new DelegateTransactionalPageOutput(finalOutput, output); aborter.dontAbort(); closer.dontClose(); // ownership of output is transferred to caller (input plugin). the owner will close output. return ret; } } }
@Override public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) { final RunnerTask task = taskSource.loadTask(RunnerTask.class); List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task); ParserPlugin parserPlugin = newParserPlugin(task); final TransactionalFileInput tran = PluginWrappers.transactionalFileInput( fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex)); try (CloseResource closer = new CloseResource(tran)) { try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) { FileInput fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), tran); closer.closeThis(fileInput); parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output); TaskReport report = tran.commit(); // TODO check output.finish() is called. wrap aborter.dontAbort(); return report; } } }
public static void process(ExecSession exec, int taskIndex, InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource, List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources, OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource, ProcessStateCallback callback) { TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput( outputPlugin.open(outputTaskSource, outputSchema, taskIndex)); callback.started(); // here needs to use try-with-resource to add exception happend at close() or abort() // to suppressed exception. otherwise exception happend at close() or abort() overwrites // essential exception. try (CloseResource closer = new CloseResource(tran)) { try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) { PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran); closer.closeThis(filtered); TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered); if (inputTaskReport == null) { inputTaskReport = exec.newTaskReport(); } callback.inputCommitted(inputTaskReport); TaskReport outputTaskReport = tran.commit(); aborter.dontAbort(); if (outputTaskReport == null) { outputTaskReport = exec.newTaskReport(); } callback.outputCommitted(outputTaskReport); // TODO check output.finish() is called. wrap or abstract } } }