@Override public void sourceConfInit( FlowProcess<? extends Config> flowProcess, Tap<Config, Input, Output> tap, Config conf ) { scheme.sourceConfInit( flowProcess, tap, conf ); }
@Override public void sourceConfInit(FlowProcess<Properties> flowProcess, Tap<Properties, RecordReader, OutputCollector> tap, Properties conf) { JobConf jobConf = mergeDefaults("LocalScheme#sourceConfInit", conf, defaults); scheme.sourceConfInit(new HadoopFlowProcess(jobConf), lfs, jobConf); overwriteProperties(conf, jobConf); }
/** * Method sourceConfInit initializes this instance as a source. * <p> * This method maybe called more than once if this Tap instance is used outside the scope of a {@link cascading.flow.Flow} * instance or if it participates in multiple times in a given Flow or across different Flows in * a {@link cascading.cascade.Cascade}. * <p> * In the context of a Flow, it will be called after * {@link cascading.flow.FlowListener#onStarting(cascading.flow.Flow)} * <p> * Note that no resources or services should be modified by this method. * * @param flowProcess of type FlowProcess * @param conf of type Config */ public void sourceConfInit( FlowProcess<? extends Config> flowProcess, Config conf ) { getScheme().sourceConfInit( flowProcess, this, conf ); }
@Override public void sourceConfInit(FlowProcess<JobConf> process, JobConf conf) { try { Path root = getQualifiedPath(conf); if (_options.attrs != null && _options.attrs.length > 0) { Pail pail = new Pail(_pailRoot, conf); for (List<String> attr : _options.attrs) { String rel = Utils.join(attr, "/"); pail.getSubPail(rel); //ensure the path exists Path toAdd = new Path(root, rel); LOG.info("Adding input path " + toAdd.toString()); FileInputFormat.addInputPath(conf, toAdd); } } else { FileInputFormat.addInputPath(conf, root); } getScheme().sourceConfInit(process, this, conf); makeLocal(conf, getQualifiedPath(conf), "forcing job to local mode, via source: "); TupleSerialization.setSerializations(conf); } catch (IOException e) { throw new TapException(e); } }