private void cleanTemporaryFiles( boolean stop ) { if( stop ) // unstable to call fs operations during shutdown return; // use step config so cascading.flow.step.path property is properly used for( FlowStep<JobConf> step : getFlowSteps() ) ( (BaseFlowStep<JobConf>) step ).clean(); }
private void cleanTemporaryFiles( boolean stop ) { if( stop ) // unstable to call fs operations during shutdown return; // use step config so cascading.flow.step.path property is properly used for( FlowStep<JobConf> step : getFlowSteps() ) ( (BaseFlowStep<JobConf>) step ).clean(); }
@Test public void testNestedProperties() throws IOException { Tap source = new Hfs( new TextLine( new Fields( "line" ) ), "/input" ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.ALL ); Tap sink = new Hfs( new TextLine(), "output", SinkMode.REPLACE ); Properties defaultProperties = new Properties(); defaultProperties.setProperty( "test.key", "test.value" ); HadoopFlow flow = (HadoopFlow) getPlatform().getFlowConnector( new Properties( defaultProperties ) ).connect( source, sink, pipe ); assertEquals( "test flow", "test.value", flow.getProperty( "test.key" ) ); assertEquals( "test step", "test.value", ( (HadoopFlowStep) flow.getFlowSteps().get( 0 ) ).createInitializedConfig( flow.getFlowProcess(), flow.getConfig() ).get( "test.key" ) ); }
@Test public void testNestedProperties() throws IOException { Tap source = new Hfs( new TextLine( new Fields( "line" ) ), "/input" ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.ALL ); Tap sink = new Hfs( new TextLine(), "output", SinkMode.REPLACE ); Properties defaultProperties = new Properties(); defaultProperties.setProperty( "test.key", "test.value" ); HadoopFlow flow = (HadoopFlow) getPlatform().getFlowConnector( new Properties( defaultProperties ) ).connect( source, sink, pipe ); assertEquals( "test flow", "test.value", flow.getProperty( "test.key" ) ); assertEquals( "test step", "test.value", ( (HadoopFlowStep) flow.getFlowSteps().get( 0 ) ).createInitializedConfig( flow.getFlowProcess(), flow.getConfig() ).get( "test.key" ) ); }