a = b = c = d = new TestFunction(); // or new TestFunction(); // or foo(new TestFunction());
public Pipe createAssembly( Pipe pipe, Fields argFields, Fields declFields, String fieldValue, Fields selectFields ) { return new Each( pipe, argFields, new TestFunction( declFields, new Tuple( fieldValue ) ), selectFields ); } }
public Pipe createAssembly( Pipe pipe, Fields argFields, Fields declFields, String fieldValue, Fields selectFields ) { return new Each( pipe, argFields, new TestFunction( declFields, new Tuple( fieldValue ) ), selectFields ); } }
@Test public void testEachPassCtor() { for( Fields arg : ARGS_PASS ) { for( Fields func : FUNCS_PASS ) { for( Fields out : OUTS_PASS ) { try { new Each( new Pipe( "test" ), arg, new TestFunction( func, new Tuple( "value" ) ), out ); } catch( Exception exception ) { fail( "failed on: " + arg.print() + " " + func.print() + " " + out.print() ); } } } } }
@Test public void testTrapNamesPass() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "map", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testTrapNamesPass2() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Pipe( "middle", pipe ); pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "middle", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testTrapNamesPass() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "map", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testTrapNamesPass2() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Pipe( "middle", pipe ); pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "middle", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testLastEachNotModified() throws Exception { copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "insert" ), new Tuple( "inserted" ) ) ); pipe = new GroupBy( pipe, new Fields( "insert" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "lasteachmodified" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10, null ); }
@Test public void testLastEachNotModified() throws Exception { copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "insert" ), new Tuple( "inserted" ) ) ); pipe = new GroupBy( pipe, new Fields( "insert" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "lasteachmodified" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10, null ); }
private Flow createRestartableFlow( String sinkPath, boolean fail ) { Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Checkpoint( pipe ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); pipe = new Each( pipe, new TestFunction( new Fields( "insert" ), new Tuple( "value" ), fail ? 2 : -1 ) ); Tap sink = getPlatform().getTextFile( sinkPath, SinkMode.REPLACE ); FlowDef flowDef = FlowDef.flowDef() .setName( "restartable" ) .addSource( "test", source ) .addTailSink( pipe, sink ) .setRunID( "restartable" ); return getPlatform().getFlowConnector().connect( flowDef ); }
private Flow createRestartableFlow( String sinkPath, boolean fail ) { Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Checkpoint( pipe ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); pipe = new Each( pipe, new TestFunction( new Fields( "insert" ), new Tuple( "value" ), fail ? 2 : -1 ) ); Tap sink = getPlatform().getTextFile( sinkPath, SinkMode.REPLACE ); FlowDef flowDef = FlowDef.flowDef() .setName( "restartable" ) .addSource( "test", source ) .addTailSink( pipe, sink ) .setRunID( "restartable" ); return getPlatform().getFlowConnector().connect( flowDef ); }
@Test public void testTrapEachAll() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "all/tap" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( getOutputPath( "all/trap" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0, null ); validateLength( flow.openTrap(), 10 ); }
@Test public void testTrapEachAllSequence() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "allseq/tap" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "allseq/trap" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0, null ); validateLength( flow.openTrap(), 10 ); }
@Test public void testTrapDiagnosticsLocalConfig() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "diagconfigdef/tap" + NONDETERMINISTIC ), SinkMode.REPLACE ); Tap trap = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "diagconfigdef/trap" + NONDETERMINISTIC ), SinkMode.REPLACE ); Map<Object, Object> properties = getProperties(); TrapProps.trapProps() .recordAllDiagnostics() .setProperties( trap.getConfigDef(), ConfigDef.Mode.DEFAULT ); Flow flow = getPlatform().getFlowConnector( properties ).connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0 ); validateLength( flow.openTrap(), 10, 4, Pattern.compile( ".*TrapPlatformTest.*" ) ); // 4 columns, not 1 }
@Test public void testTrapEachAllSequence() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "allseq/tap" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "allseq/trap" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0, null ); validateLength( flow.openTrap(), 10 ); }
@Test public void testTrapEachAll() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "all/tap" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( getOutputPath( "all/trap" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0, null ); validateLength( flow.openTrap(), 10 ); }
@Test public void testTrapToSequenceFile() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "seq/tap" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( "seq/trap" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0, null ); validateLength( flow.openTrap(), 10 ); }
@Test public void testTrapToSequenceFile() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "seq/tap" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( "seq/trap" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 0, null ); validateLength( flow.openTrap(), 10 ); }
@Test(expected = CascadingException.class) public void testTrapFailure() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Scheme scheme = getPlatform().getTestFailScheme(); Tap trap2 = getPlatform().getTap( scheme, getOutputPath( "trapFailure/badTrap" ), SinkMode.REPLACE ); Tap sink = getPlatform().getTextFile( getOutputPath( "trapFailure/tap" ), SinkMode.REPLACE ); Pipe pipe = new Each( new Pipe( "firstPipe" ), new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Each( new Pipe( "secondPipe", pipe ), new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); Tap trap1 = getPlatform().getTextFile( getOutputPath( "trapFailure/firstTrap" ), SinkMode.REPLACE ); FlowDef flowDef = FlowDef.flowDef() .addSource( "firstPipe", source ) .addTrap( "firstPipe", trap1 ) .addTrap( "secondPipe", trap2 ) .addTail( pipe ) .addSink( pipe, sink ); Flow flow = getPlatform().getFlowConnector().connect( flowDef ); flow.complete(); } }