public TaskReport commit() { TaskReport report = Exec.newTaskReport(); // TODO better setting for Report // report.set("file_names", fileNames); // report.set("file_sizes", fileSizes); return report; } };
public void setOutputConfigDiff(ConfigDiff outputConfigDiff) { if (outputConfigDiff == null) { outputConfigDiff = Exec.newConfigDiff(); } this.outputConfigDiff = outputConfigDiff; }
@Deprecated public ColumnConfig(String name, Type type, String format) { this.name = name; this.type = type; this.option = Exec.newConfigSource(); // only for backward compatibility if (format != null) { option.set("format", format); } }
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { control.run(Exec.newTaskSource(), 1); return Exec.newConfigDiff(); }
final JsonParser jsonParser = new JsonParser(); try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) { for (final List<JsonNode> rowValues : taskValues) { schema.visitColumns(new ColumnVisitor() { return Exec.newTaskReport();
@Override public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput pageOutput) { PluginTask task = taskSource.loadTask(PluginTask.class); final ConfigSource originalConfig = task.getOriginalConfig(); final int guessParserSampleBufferBytes = task.getGuessParserSampleBufferBytes(); // get sample buffer Buffer sample = readSample(input, guessParserSampleBufferBytes); // load guess plugins ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder(); for (PluginType guessType : task.getGuessPluginTypes()) { GuessPlugin guess = Exec.newPlugin(GuessPlugin.class, guessType); builder.add(guess); } List<GuessPlugin> guesses = builder.build(); // run guess plugins ConfigSource mergedConfig = originalConfig.deepCopy(); ConfigDiff mergedGuessed = Exec.newConfigDiff(); for (int i = 0; i < guesses.size(); i++) { ConfigDiff guessed = guesses.get(i).guess(originalConfig, sample); guessed = addAssumedDecoderConfigs(originalConfig, guessed); mergedGuessed.merge(guessed); mergedConfig.merge(mergedGuessed); if (!mergedConfig.equals(originalConfig)) { // config updated throw new GuessedNoticeError(mergedGuessed); } } throw new GuessedNoticeError(mergedGuessed); }
private PageBuilder newPageBuilder(Schema schema, PageOutput output) { return new PageBuilder(Exec.getBufferAllocator(), schema, output); }
public ElasticsearchHttpClient() { this.log = Exec.getLogger(getClass()); }
@Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) { return resume(Exec.newTaskSource(), schema, taskCount, control); }
public ExecutionResult run(ExecSession exec, final ConfigSource config) { try { return Exec.doWith(exec, new ExecAction<ExecutionResult>() { public ExecutionResult run() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName("transaction")) { return doRun(config); } } }); } catch (ExecutionException ex) { if (ex.getCause() instanceof RuntimeException) { throw (RuntimeException) ex.getCause(); } if (ex.getCause() instanceof Error) { throw (Error) ex.getCause(); } throw new RuntimeException(ex.getCause()); } }
@Override public PageOutput open(TaskSource taskSource, Schema inputSchema, Schema outputSchema, PageOutput output) { PluginTask task = taskSource.loadTask(PluginTask.class); PageReader pageReader = new PageReader(inputSchema); PageBuilder pageBuilder = new PageBuilder(getBufferAllocator(), outputSchema, output); return new PageConverter(pageReader, pageBuilder, task.getIndexMapping()); }
public ElasticsearchOutputPluginDelegate() { this.log = Exec.getLogger(getClass()); this.client = new ElasticsearchHttpClient(); }
@Override public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { control.run(Exec.newTaskSource(), 1); return null; }
public ConfigDiff guess(ExecSession exec, final ConfigSource config) { try { return Exec.doWith(exec, new ExecAction<ConfigDiff>() { public ConfigDiff run() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName("guess")) { return doGuess(config); } } }); } catch (ExecutionException ex) { if (ex.getCause() instanceof RuntimeException) { throw (RuntimeException) ex.getCause(); } if (ex.getCause() instanceof Error) { throw (Error) ex.getCause(); } throw new RuntimeException(ex.getCause()); } }
@Override public ConfigDiff guess(ConfigSource config) { return guess(Exec.newConfigSource(), config); }
public TaskReport commit() { return Exec.newTaskReport(); } };
@Override public ConfigDiff guess(ConfigSource config) { return Exec.newConfigDiff(); } }
final int skipHeaderLines = task.getSkipHeaderLines(); try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) { while (tokenizer.nextFile()) { final String fileName = input.hintOfCurrentInputFileNameForLogging().orElse("-");
public ElasticsearchRecordBuffer(String attributeName, PluginTask task) { this.attributeName = attributeName; this.task = task; this.bulkActions = task.getBulkActions(); this.bulkSize = task.getBulkSize(); this.client = new ElasticsearchHttpClient(); this.mapper = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false); this.records = JsonNodeFactory.instance.arrayNode(); this.totalCount = 0; this.requestCount = 0; this.requestBytes = 0; this.log = Exec.getLogger(getClass()); }
public ProcessTask buildProcessTask() { return new ProcessTask( plugins.getInputPluginType(), plugins.getOutputPluginType(), plugins.getFilterPluginTypes(), inputTaskSource, outputTaskSource, filterTaskSources, schemas, executorSchema, Exec.newTaskSource()); }