a = b = c = d = new TestFunction(); // or new TestFunction(); // or foo(new TestFunction());
public void operate( FlowProcess flowProcess, FunctionCall<Integer> functionCall ) { if( value == null ) throwIntentionalException(); try { if( functionCall.getContext() == failon ) throw new RuntimeException( "function failed intentionally on tuple number: " + failon ); } finally { functionCall.setContext( functionCall.getContext() + 1 ); } TupleEntry result; if( functionCall.getDeclaredFields().isUnknown() ) result = new TupleEntry( Fields.size( value.size() ), Tuple.size( value.size() ) ); else result = new TupleEntry( functionCall.getDeclaredFields(), Tuple.size( functionCall.getDeclaredFields().size() ) ); try { result.setCanonicalTuple( value ); } catch( Exception exception ) { // value is a string, but the the test declared a numeric type result.setCanonicalTuple( Tuple.size( value.size(), -99 ) ); } functionCall.getOutputCollector().add( result ); }
public Pipe createAssembly( Pipe pipe, Fields argFields, Fields declFields, String fieldValue, Fields selectFields ) { return new Each( pipe, argFields, new TestFunction( declFields, new Tuple( fieldValue ) ), selectFields ); } }
public Pipe createAssembly( Pipe pipe, Fields argFields, Fields declFields, String fieldValue, Fields selectFields ) { return new Each( pipe, argFields, new TestFunction( declFields, new Tuple( fieldValue ) ), selectFields ); } }
Pipe result = new Each(resultsTwo, functionFields, new TestFunction(), Fields.REPLACE);
Pipe result = new Each(resultsTwo, functionFields, new TestFunction(), Fields.REPLACE);
@Test public void testEachPassCtor() { for( Fields arg : ARGS_PASS ) { for( Fields func : FUNCS_PASS ) { for( Fields out : OUTS_PASS ) { try { new Each( new Pipe( "test" ), arg, new TestFunction( func, new Tuple( "value" ) ), out ); } catch( Exception exception ) { fail( "failed on: " + arg.print() + " " + func.print() + " " + out.print() ); } } } } }
new Each( new Pipe( "test" ), arg, new TestFunction( func, new Tuple( "value" ) ), out ); fail( "failed on: " + arg.print() + " " + func.print() + " " + out.print() ); new Each( new Pipe( "test" ), arg, new TestFunction( func, new Tuple( "value" ) ), out ); fail( "failed on: " + arg.print() + " " + func.print() + " " + out.print() ); new Each( new Pipe( "test" ), arg, new TestFunction( func, new Tuple( "value" ) ), out ); fail( "failed on: " + arg.print() + " " + func.print() + " " + out.print() );
/** * verify we can fail in randome places into the same trap * * @throws Exception */ @Test public void testTrapEachAllChained() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new TestFunction( new Fields( "test" ), new Tuple( 1 ), 1 ), Fields.ALL ); pipe = new Each( pipe, new TestFunction( new Fields( "test2" ), new Tuple( 2 ), 2 ), Fields.ALL ); pipe = new Each( pipe, new TestFunction( new Fields( "test3" ), new Tuple( 3 ), 3 ), Fields.ALL ); pipe = new Each( pipe, new TestFunction( new Fields( "test4" ), new Tuple( 4 ), 4 ), Fields.ALL ); Tap sink = getPlatform().getTextFile( getOutputPath( "allchain/tap-nondeterministic" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( getOutputPath( "allchain/trap-nondeterministic" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 6, null ); validateLength( flow.openTrap(), 4 ); }
@Test public void testTrapNamesPass() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "map", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testTrapNamesPass() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "map", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testTrapNamesPass2() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Pipe( "middle", pipe ); pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "middle", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
@Test public void testTrapNamesPass2() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new Pipe( "middle", pipe ); pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "foosink" ); Tap trap = getPlatform().getTextFile( "footrap" ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "map", source ); sinks.put( "reduce", sink ); traps.put( "middle", trap ); getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); }
/** * verify we can fail in randome places into the same trap * * @throws Exception */ @Test public void testTrapEachAllChained() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new TestFunction( new Fields( "test" ), new Tuple( 1 ), 1 ), Fields.ALL ); pipe = new Each( pipe, new TestFunction( new Fields( "test2" ), new Tuple( 2 ), 2 ), Fields.ALL ); pipe = new Each( pipe, new TestFunction( new Fields( "test3" ), new Tuple( 3 ), 3 ), Fields.ALL ); pipe = new Each( pipe, new TestFunction( new Fields( "test4" ), new Tuple( 4 ), 4 ), Fields.ALL ); Tap sink = getPlatform().getTextFile( getOutputPath( "allchain/tap-nondeterministic" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( getOutputPath( "allchain/trap-nondeterministic" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow, 6, null ); validateLength( flow.openTrap(), 4 ); }
@Test public void testTrapNamesFail() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "footap", SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( "footrap", SinkMode.REPLACE ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "test", source ); sinks.put( "test", sink ); traps.put( "nada", trap ); try { getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); fail( "did not fail on missing pipe name" ); } catch( Exception exception ) { // tests passed } }
@Test public void testSplitOnNonSafeOperationsSimple() { Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" ); Tap sink1 = new Hfs( new TextLine(), "foo/split1", SinkMode.REPLACE ); Tap sink2 = new Hfs( new TextLine(), "foo/split2", SinkMode.REPLACE ); Pipe pipe = new Pipe( "split" ); // this operation is not safe pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) ); Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) ); Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*192.*" ) ); Map sources = new HashMap(); sources.put( "split", source ); Map sinks = new HashMap(); sinks.put( "left", sink1 ); sinks.put( "right", sink2 ); Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "not equal: steps.size()", 3, steps.size() ); FlowStep step = steps.get( 0 ); assertEquals( "wrong number of operations", 1, ( (BaseFlowStep) step ).getAllOperations().size() ); }
@Test public void testSplitOnNonSafeOperationsSimple() { Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" ); Tap sink1 = new Hfs( new TextLine(), "foo/split1", SinkMode.REPLACE ); Tap sink2 = new Hfs( new TextLine(), "foo/split2", SinkMode.REPLACE ); Pipe pipe = new Pipe( "split" ); // this operation is not safe pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) ); Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) ); Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*192.*" ) ); Map sources = new HashMap(); sources.put( "split", source ); Map sinks = new HashMap(); sinks.put( "left", sink1 ); sinks.put( "right", sink2 ); Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "not equal: steps.size()", 3, steps.size() ); FlowStep step = steps.get( 0 ); assertEquals( "wrong number of operations", 1, ( (BaseFlowStep) step ).getAllOperations().size() ); }
@Test public void testTrapNamesFail() throws Exception { Tap source = getPlatform().getTextFile( "foosource" ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); // always fail pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( "footap", SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( "footrap", SinkMode.REPLACE ); Map<String, Tap> sources = new HashMap<String, Tap>(); Map<String, Tap> sinks = new HashMap<String, Tap>(); Map<String, Tap> traps = new HashMap<String, Tap>(); sources.put( "test", source ); sinks.put( "test", sink ); traps.put( "nada", trap ); try { getPlatform().getFlowConnector().connect( "trap test", sources, sinks, traps, pipe ); fail( "did not fail on missing pipe name" ); } catch( Exception exception ) { // tests passed } }
@Test public void testSplitOnNonSafeOperations() { Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" ); Tap sink1 = new Hfs( new TextLine(), "foo/split1", SinkMode.REPLACE ); Tap sink2 = new Hfs( new TextLine(), "foo/split2", SinkMode.REPLACE ); Pipe pipe = new Pipe( "split" ); // this operation is not safe pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) ); pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) ); Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) ); Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*192.*" ) ); Map sources = new HashMap(); sources.put( "split", source ); Map sinks = new HashMap(); sinks.put( "left", sink1 ); sinks.put( "right", sink2 ); Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "not equal: steps.size()", 3, steps.size() ); FlowStep step = steps.get( 0 ); assertEquals( "wrong number of operations", 2, ( (BaseFlowStep) step ).getAllOperations().size() ); }
@Test public void testOOMEPreGroup() throws Exception { copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "insert" ), null ) { @Override protected void throwIntentionalException() { throw new OutOfMemoryError( "fake error" ); } } ); Tap sink = getPlatform().getTextFile( getOutputPath( "oomepre" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); try { flow.complete(); fail( "no failure thrown" ); } catch( Throwable exception ) { System.out.println( "exception = " + exception ); exception.printStackTrace(); // test local mode passes the OOME on, other its squashed, but the flow should fail if( !getPlatform().isMapReduce() && !getPlatform().isDAG() ) assertTrue( exception instanceof OutOfMemoryError ); } }