@Override protected FlowPlanner createFlowPlanner() { return new Hadoop2TezPlanner(); }
@Override protected Hadoop2TezFlow createFlow( FlowDef flowDef ) { return new Hadoop2TezFlow( getPlatformInfo(), getDefaultProperties(), getDefaultConfig(), flowDef ); }
@Override public FlowStepFactory<TezConfiguration> getFlowStepFactory() { return new BaseFlowStepFactory<TezConfiguration>( getFlowNodeFactory() ) { @Override public FlowStep<TezConfiguration> createFlowStep( ElementGraph stepElementGraph, FlowNodeGraph flowNodeGraph ) { return new Hadoop2TezFlowStep( stepElementGraph, flowNodeGraph ); } }; }
Hadoop2TezPlanner.copyProperties( configuration, globalProperties ); // copy any external properties Hadoop2TezPlanner.copyConfiguration( properties, configuration ); // put all properties on the jobconf
public static TezConfiguration createConfiguration( Map<Object, Object> properties ) { TezConfiguration conf = new TezConfiguration(); copyProperties( conf, properties ); return conf; }
@Override public void configRuleRegistryDefaults( RuleRegistry ruleRegistry ) { super.configRuleRegistryDefaults( ruleRegistry ); ruleRegistry.addDefaultElementFactory( IntermediateTapElementFactory.TEMP_TAP, new TempTapElementFactory() ); ruleRegistry.addDefaultElementFactory( BoundaryElementFactory.BOUNDARY_PIPE, new IntermediateBoundaryElementFactory() ); if( PropertyUtil.getBooleanProperty( getDefaultProperties(), FlowConnectorProps.ENABLE_DECORATE_ACCUMULATED_TAP, true ) ) ruleRegistry.addDefaultElementFactory( IntermediateTapElementFactory.ACCUMULATED_TAP, new TempTapElementFactory( DistCacheTap.class.getName() ) ); }
@Override public void initialize( FlowConnector flowConnector, Map<Object, Object> properties ) { super.initialize( flowConnector, properties ); defaultConfiguration = TezUtil.createTezConf( properties, createConfiguration( properties ) ); intermediateSchemeClass = flowConnector.getIntermediateSchemeClass( properties ); String applicationJarPath = AppProps.getApplicationJarPath( properties ); if( applicationJarPath == null ) { Class type = AppProps.getApplicationJarClass( properties ); if( type == null ) type = HadoopUtil.findMainClass( Hadoop2TezPlanner.class ); if( type != null ) applicationJarPath = Util.findContainingJar( type ); AppProps.setApplicationJarPath( properties, applicationJarPath ); } if( applicationJarPath != null ) LOG.info( "using application jar: {}", applicationJarPath ); else LOG.info( "using application jar not provided, see cascading.property.AppProps for more information" ); }
Hadoop2TezPlanner.copyProperties( configuration, globalProperties ); // copy any external properties Hadoop2TezPlanner.copyConfiguration( properties, configuration ); // put all properties on the jobconf
public static TezConfiguration createConfiguration( Map<Object, Object> properties ) { TezConfiguration conf = new TezConfiguration(); copyProperties( conf, properties ); return conf; }
@Override public void configRuleRegistryDefaults( RuleRegistry ruleRegistry ) { super.configRuleRegistryDefaults( ruleRegistry ); ruleRegistry.addDefaultElementFactory( IntermediateTapElementFactory.TEMP_TAP, new TempTapElementFactory() ); ruleRegistry.addDefaultElementFactory( BoundaryElementFactory.BOUNDARY_PIPE, new IntermediateBoundaryElementFactory() ); if( PropertyUtil.getBooleanProperty( getDefaultProperties(), FlowConnectorProps.ENABLE_DECORATE_ACCUMULATED_TAP, true ) ) ruleRegistry.addDefaultElementFactory( IntermediateTapElementFactory.ACCUMULATED_TAP, new TempTapElementFactory( DistCacheTap.class.getName() ) ); }
@Override public void initialize( FlowConnector flowConnector, Map<Object, Object> properties ) { super.initialize( flowConnector, properties ); defaultConfiguration = TezUtil.createTezConf( properties, createConfiguration( properties ) ); intermediateSchemeClass = flowConnector.getIntermediateSchemeClass( properties ); String applicationJarPath = AppProps.getApplicationJarPath( properties ); if( applicationJarPath == null ) { Class type = AppProps.getApplicationJarClass( properties ); if( type == null ) type = HadoopUtil.findMainClass( Hadoop2TezPlanner.class ); if( type != null ) applicationJarPath = Util.findContainingJar( type ); AppProps.setApplicationJarPath( properties, applicationJarPath ); } if( applicationJarPath != null ) LOG.info( "using application jar: {}", applicationJarPath ); else LOG.info( "using application jar not provided, see cascading.property.AppProps for more information" ); }
@Override protected Hadoop2TezFlow createFlow( FlowDef flowDef ) { return new Hadoop2TezFlow( getPlatformInfo(), getDefaultProperties(), getDefaultConfig(), flowDef ); }
@Override public FlowStepFactory<TezConfiguration> getFlowStepFactory() { return new BaseFlowStepFactory<TezConfiguration>( getFlowNodeFactory() ) { @Override public FlowStep<TezConfiguration> createFlowStep( ElementGraph stepElementGraph, FlowNodeGraph flowNodeGraph ) { return new Hadoop2TezFlowStep( stepElementGraph, flowNodeGraph ); } }; }
@Override protected FlowPlanner createFlowPlanner() { return new Hadoop2TezPlanner(); }