@Override protected void addPropertiesTo( Properties properties ) { setDefaultTupleElementComparator( properties, defaultTupleElementComparator ); setPreserveTemporaryFiles( properties, preserveTemporaryFiles ); setJobPollingInterval( properties, jobPollingInterval ); setMaxConcurrentSteps( properties, maxConcurrentSteps ); setStopJobsOnExit( properties, stopJobsOnExit ); } }
/** * Creates a new FlowProps instance. * * @return FlowProps instance */ public static FlowProps flowProps() { return new FlowProps(); }
globalProperties.put( "log4j.logger", logger ); FlowProps.setJobPollingInterval( globalProperties, 10 ); // should speed up tests
FlowProps.setJobPollingInterval(props, jobPoolingInterval); FlowProps.setMaxConcurrentSteps(props, maxConcurrentSteps);
@Test public void testCoGroup() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Tap sourceLower = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileLower ); Tap sourceUpper = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileUpper ); Map sources = new HashMap(); sources.put( "lower", sourceLower ); sources.put( "upper", sourceUpper ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "cogroup" ), SinkMode.REPLACE ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter ); Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter ); Pipe splice = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), new InnerJoin( Fields.size( 4 ) ) ); Map<Object, Object> properties = getProperties(); // make sure hasher is getting called, but does nothing special FlowProps.setDefaultTupleElementComparator( properties, getPlatform().getStringComparator( false ).getClass().getCanonicalName() ); Flow flow = getPlatform().getFlowConnector( properties ).connect( sources, sink, splice ); flow.complete(); validateLength( flow, 5 ); List<Tuple> values = getSinkAsList( flow ); assertTrue( values.contains( new Tuple( "1\ta\t1\tA" ) ) ); assertTrue( values.contains( new Tuple( "2\tb\t2\tB" ) ) ); }
globalProperties.put( "log4j.logger", logger ); FlowProps.setJobPollingInterval( globalProperties, 10 ); // should speed up tests
@Test public void testCoGroup() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Tap sourceLower = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileLower ); Tap sourceUpper = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileUpper ); Map sources = new HashMap(); sources.put( "lower", sourceLower ); sources.put( "upper", sourceUpper ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "cogroup" ), SinkMode.REPLACE ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter ); Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter ); Pipe splice = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), new InnerJoin( Fields.size( 4 ) ) ); Map<Object, Object> properties = getProperties(); // make sure hasher is getting called, but does nothing special FlowProps.setDefaultTupleElementComparator( properties, getPlatform().getStringComparator( false ).getClass().getCanonicalName() ); Flow flow = getPlatform().getFlowConnector( properties ).connect( sources, sink, splice ); flow.complete(); validateLength( flow, 5 ); List<Tuple> values = getSinkAsList( flow ); assertTrue( values.contains( new Tuple( "1\ta\t1\tA" ) ) ); assertTrue( values.contains( new Tuple( "2\tb\t2\tB" ) ) ); }
globalProperties.put( "log4j.logger", logger ); FlowProps.setJobPollingInterval( globalProperties, 10 ); // should speed up tests
globalProperties.put( "log4j.logger", logger ); FlowProps.setJobPollingInterval( globalProperties, 10 ); // should speed up tests