@Override public void prepareRun(BatchSinkContext context) throws Exception { OutputFormatProvider outputFormatProvider = new BasicOutputFormatProvider(TextOutputFormat.class.getCanonicalName(), ImmutableMap.of(TextOutputFormat.OUTDIR, config.dirName)); if (config.name != null) { Output output = Output.of(config.name, outputFormatProvider); output.alias(config.alias); context.addOutput(output); } else { context.addOutput(Output.of(config.alias, outputFormatProvider)); } }
/** * Get the output, if preview is enabled, return the output with a {@link NullOutputFormatProvider}. */ private Output getOutput(Output output) { if (isPreviewEnabled) { return Output.of(output.getName(), new NullOutputFormatProvider()); } return output; } }
@Override public final void prepareRun(BatchSinkContext context) throws InstantiationException { config.validate(); // set format specific properties. OutputFormatProvider outputFormatProvider = context.newPluginInstance(FORMAT_PLUGIN_ID); // record field level lineage information // needs to happen before context.addOutput(), otherwise an external dataset without schema will be created. Schema schema = config.getSchema(); if (schema == null) { schema = context.getInputSchema(); } LineageRecorder lineageRecorder = new LineageRecorder(context, config.getReferenceName()); lineageRecorder.createExternalDataset(schema); if (schema != null && schema.getFields() != null && !schema.getFields().isEmpty()) { recordLineage(lineageRecorder, schema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList())); } Map<String, String> outputProperties = new HashMap<>(outputFormatProvider.getOutputFormatConfiguration()); outputProperties.putAll(getFileSystemProperties(context)); outputProperties.put(FileOutputFormat.OUTDIR, getOutputDir(context.getLogicalStartTime())); context.addOutput(Output.of(config.getReferenceName(), new SinkOutputFormatProvider(outputFormatProvider.getOutputFormatClassName(), outputProperties))); }
context.addOutput(Output.of("test", new OutputFormatProvider() { @Override public String getOutputFormatClassName() {
@Override public void initialize() throws Exception { // this write should be invalidated if any of the following fails KeyValueTable kvTable = getContext().getDataset("recorder"); kvTable.write("initialized", "true"); if (getContext().getRuntimeArguments().containsKey("failInput")) { getContext().addInput(Input.of("x", new FailingInputFormatProvider())); } if (getContext().getRuntimeArguments().containsKey("failOutput")) { getContext().addOutput(Output.of("x", new FailingOutputFormatProvider())); } } }