public void setOutputConfigDiff(ConfigDiff outputConfigDiff) { if (outputConfigDiff == null) { outputConfigDiff = Exec.newConfigDiff(); } this.outputConfigDiff = outputConfigDiff; }
@Override public ConfigDiff guess(ConfigSource config) { return Exec.newConfigDiff(); } }
public void setInputConfigDiff(ConfigDiff inputConfigDiff) { if (inputConfigDiff == null) { inputConfigDiff = Exec.newConfigDiff(); } this.inputConfigDiff = inputConfigDiff; }
@Override public ConfigDiff guess(ConfigSource config) { return Exec.newConfigDiff(); } }
@Override public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, OutputPlugin.Control control) { control.run(taskSource); return Exec.newConfigDiff(); }
@Override public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, Control control) { control.run(taskSource, schema, taskCount); return Exec.newConfigDiff(); }
@Override public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control) { control.run(taskSource, schema, taskCount); return Exec.newConfigDiff(); }
public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, OutputPlugin.Control control) { control.run(taskSource); return Exec.newConfigDiff(); }
@Override public ConfigDiff resume(TaskSource taskSource, int taskCount, FileOutputPlugin.Control control) { control.run(taskSource); return Exec.newConfigDiff(); }
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { control.run(Exec.newTaskSource(), 1); return Exec.newConfigDiff(); }
@Override public ConfigDiff resume(final TaskSource taskSource, final int taskCount, final FileInputPlugin.Control control) { final PluginTask task = taskSource.loadTask(PluginTask.class); control.run(taskSource, taskCount); // build next config final ConfigDiff configDiff = Exec.newConfigDiff(); // last_path if (task.getFiles().isEmpty()) { // keep the last value if (task.getLastPath().isPresent()) { configDiff.set("last_path", task.getLastPath().get()); } } else { final List<String> files = new ArrayList<String>(task.getFiles()); Collections.sort(files); configDiff.set("last_path", files.get(files.size() - 1)); } return configDiff; }
public ExecutionResult buildExecuteResultWithWarningException(Throwable ex) { ConfigDiff configDiff = Exec.newConfigDiff(); if (inputConfigDiff != null) { configDiff.getNestedOrSetEmpty("in").merge(inputConfigDiff); } if (outputConfigDiff != null) { configDiff.getNestedOrSetEmpty("out").merge(outputConfigDiff); } ImmutableList.Builder<Throwable> ignoredExceptions = ImmutableList.builder(); for (Throwable e : getExceptions()) { ignoredExceptions.add(e); } if (ex != null) { ignoredExceptions.add(ex); } return new ExecutionResult(configDiff, false, ignoredExceptions.build()); }
private ConfigDiff doGuess(ConfigSource config) { ConfigSource inputConfig = config.getNested("in"); ConfigSource execConfig = config.getNestedOrGetEmpty("exec"); InputPlugin input = newInputPlugin(inputConfig); ConfigDiff inputGuessed; if (input instanceof ConfigurableGuessInputPlugin) { inputGuessed = ((ConfigurableGuessInputPlugin) input).guess(execConfig, inputConfig); } else { try { inputGuessed = input.guess(inputConfig); } catch (AbstractMethodError ex) { // for backward compatibility with embulk v0.4 interface throw new UnsupportedOperationException(input.getClass().getSimpleName() + ".guess(ConfigSource) is not implemented. This input plugin does not support guessing."); } } ConfigDiff wrapped = Exec.newConfigDiff(); wrapped.getNestedOrSetEmpty("in").merge(inputGuessed); return wrapped; }
@Override public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput pageOutput) { PluginTask task = taskSource.loadTask(PluginTask.class); final ConfigSource originalConfig = task.getOriginalConfig(); final int guessParserSampleBufferBytes = task.getGuessParserSampleBufferBytes(); // get sample buffer Buffer sample = readSample(input, guessParserSampleBufferBytes); // load guess plugins ImmutableList.Builder<GuessPlugin> builder = ImmutableList.builder(); for (PluginType guessType : task.getGuessPluginTypes()) { GuessPlugin guess = Exec.newPlugin(GuessPlugin.class, guessType); builder.add(guess); } List<GuessPlugin> guesses = builder.build(); // run guess plugins ConfigSource mergedConfig = originalConfig.deepCopy(); ConfigDiff mergedGuessed = Exec.newConfigDiff(); for (int i = 0; i < guesses.size(); i++) { ConfigDiff guessed = guesses.get(i).guess(originalConfig, sample); guessed = addAssumedDecoderConfigs(originalConfig, guessed); mergedGuessed.merge(guessed); mergedConfig.merge(mergedGuessed); if (!mergedConfig.equals(originalConfig)) { // config updated throw new GuessedNoticeError(mergedGuessed); } } throw new GuessedNoticeError(mergedGuessed); }
@Override public ConfigDiff transaction(ConfigSource config, int taskCount, FileOutputPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); control.run(task.dump()); return Exec.newConfigDiff(); }
private ConfigDiff guessParserConfig(Buffer sample, ConfigSource config, List<PluginType> guessPlugins, final int guessParserSampleBufferBytes) { ConfigDiff lastGuessed = Exec.newConfigDiff(); for (int i = 0; i < 10; i++) {
public ConfigDiff guess(ConfigSource config) { return Exec.newConfigDiff(); }
protected ConfigDiff buildNextConfigDiff(PluginTask task, List<TaskReport> reports) { ConfigDiff next = Exec.newConfigDiff(); if (reports.size() > 0 && reports.get(0).has("last_record")) { next.set("last_record", reports.get(0).get(JsonNode.class, "last_record")); } else if (task.getLastRecord().isPresent()) { next.set("last_record", task.getLastRecord().get()); } return next; }
@Override public ConfigDiff egestEmbulkData(final PluginTask task, Schema schema, int taskIndex, List<TaskReport> taskReports) { long totalInserted = 0; for (TaskReport taskReport : taskReports) { if (taskReport.has("inserted")) { totalInserted += taskReport.get(Long.class, "inserted"); } } log.info("Insert completed. {} records", totalInserted); // Re assign alias only when repale mode if (task.getMode().equals(Mode.REPLACE)) { client.reassignAlias(task.getAlias().orElse(null), task.getIndex(), task); } return Exec.newConfigDiff(); } }
@Override public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) { PluginTask task = taskSource.loadTask(getTaskClass()); // validate task newS3Client(task); control.run(taskSource, taskCount); // build next config ConfigDiff configDiff = Exec.newConfigDiff(); // last_path if (task.getIncremental()) { Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath()); LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse("")); configDiff.set("last_path", lastPath); } return configDiff; }