@Override public void prepareRun(BatchSourceContext context) throws Exception { context.setInput(Input.of(config.name, new InputFormatProvider() { @Override public String getInputFormatClassName() { return TextInputFormat.class.getCanonicalName(); } @Override public Map<String, String> getInputFormatConfiguration() { return ImmutableMap.of(TextInputFormat.INPUT_DIR, config.dirName); } })); }
context.setInput(Input.of(config.referenceName, new SourceInputFormatProvider(XMLInputFormat.class, conf)));
context.setInput(Input.of(config.getReferenceName(), new SourceInputFormatProvider(inputFormatClass, conf)));
@Override public void prepareRun(BatchSourceContext batchSourceContext) throws Exception { excelInputreaderConfig.validate(); createDatasets(null, batchSourceContext); Job job = JobUtils.createInstance(); String processFiles = ""; if (!Strings.isNullOrEmpty(excelInputreaderConfig.memoryTableName)) { processFiles = GSON.toJson(getAllProcessedFiles(batchSourceContext), ARRAYLIST_PREPROCESSED_FILES); } ExcelInputFormat.setConfigurations(job, excelInputreaderConfig.filePattern, excelInputreaderConfig.sheet, excelInputreaderConfig.reprocess, excelInputreaderConfig.sheetValue, excelInputreaderConfig.columnList, excelInputreaderConfig.skipFirstRow, excelInputreaderConfig.terminateIfEmptyRow, excelInputreaderConfig.rowsLimit, excelInputreaderConfig.ifErrorRecord, processFiles); // Sets the input path(s). ExcelInputFormat.addInputPaths(job, excelInputreaderConfig.filePath); // Sets the filter based on extended class implementation. ExcelInputFormat.setInputPathFilter(job, ExcelReaderRegexFilter.class); SourceInputFormatProvider inputFormatProvider = new SourceInputFormatProvider(ExcelInputFormat.class, job.getConfiguration()); batchSourceContext.setInput(Input.of(excelInputreaderConfig.referenceName, inputFormatProvider)); }
sourceConf.put(ADDITIONAL_CONFIG, SOURCE_CONFIG); context.addInput(Input.of("input", new InputFormatProvider() { @Override public String getInputFormatClassName() {
private Input.InputFormatProviderInput createInput(Input.DatasetInput datasetInput) { String datasetName = datasetInput.getName(); Map<String, String> datasetArgs = datasetInput.getArguments(); // keep track of the original alias to set it on the created Input before returning it String originalAlias = datasetInput.getAlias(); Dataset dataset; if (datasetInput.getNamespace() == null) { dataset = getDataset(datasetName, datasetArgs, AccessType.READ); } else { dataset = getDataset(datasetInput.getNamespace(), datasetName, datasetArgs, AccessType.READ); } DatasetInputFormatProvider datasetInputFormatProvider = new DatasetInputFormatProvider(datasetInput.getNamespace(), datasetName, datasetArgs, dataset, datasetInput.getSplits(), MapReduceBatchReadableInputFormat.class); return (Input.InputFormatProviderInput) Input.of(datasetName, datasetInputFormatProvider).alias(originalAlias); }
private Input.InputFormatProviderInput createInput(Input.DatasetInput datasetInput) { String datasetName = datasetInput.getName(); Map<String, String> datasetArgs = datasetInput.getArguments(); // keep track of the original alias to set it on the created Input before returning it String originalAlias = datasetInput.getAlias(); Dataset dataset; if (datasetInput.getNamespace() == null) { dataset = getDataset(datasetName, datasetArgs, AccessType.READ); } else { dataset = getDataset(datasetInput.getNamespace(), datasetName, datasetArgs, AccessType.READ); } DatasetInputFormatProvider datasetInputFormatProvider = new DatasetInputFormatProvider(datasetInput.getNamespace(), datasetName, datasetArgs, dataset, datasetInput.getSplits(), MapReduceBatchReadableInputFormat.class); return (Input.InputFormatProviderInput) Input.of(datasetName, datasetInputFormatProvider).alias(originalAlias); }
@Override public void initialize() throws Exception { // this write should be invalidated if any of the following fails KeyValueTable kvTable = getContext().getDataset("recorder"); kvTable.write("initialized", "true"); if (getContext().getRuntimeArguments().containsKey("failInput")) { getContext().addInput(Input.of("x", new FailingInputFormatProvider())); } if (getContext().getRuntimeArguments().containsKey("failOutput")) { getContext().addOutput(Output.of("x", new FailingOutputFormatProvider())); } } }