public TestBuffer getBuffer() { TestBuffer buffer = null; if (colorToBufferMap.containsKey(defaultColor)) { buffer = colorToBufferMap.get(defaultColor); } else { buffer = new TestBuffer(); colorToBufferMap.put(defaultColor, buffer); } return buffer; }
Pipe left = new Every( new Pipe( "output1", pipe ), new TestBuffer( new Fields( "left" ), true ) ); Pipe right = new Every( new Pipe( "output2", pipe ), new TestBuffer( new Fields( "right" ), true ) );
Pipe left = new Every( new Pipe( "output1", pipe ), new TestBuffer( new Fields( "left" ), true ) ); Pipe right = new Every( new Pipe( "output2", pipe ), new TestBuffer( new Fields( "right" ), true ) );
@Test public void testBuffer() throws IOException { Map sources = new HashMap(); Map sinks = new HashMap(); sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) ); sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) ); List steps = getPlatform().getFlowConnector().connect( sources, sinks, pipe ).getFlowSteps(); assertEquals( "wrong size", 1, steps.size() ); BaseFlowStep step = (BaseFlowStep) steps.get( 0 ); assertEquals( "not equal: step.sources.size()", 1, step.getSourceTaps().size() ); assertNotNull( "null: step.groupBy", step.getGroup() ); assertNotNull( "null: step.sink", step.getSink() ); int mapDist = ElementGraphs.shortestDistance( step.getElementGraph(), (FlowElement) step.getSourceTaps().iterator().next(), step.getGroup() ); assertEquals( "not equal: mapDist", 1, mapDist ); int reduceDist = ElementGraphs.shortestDistance( step.getElementGraph(), step.getGroup(), step.getSink() ); assertEquals( "not equal: reduceDist", 2, reduceDist ); }
@Test public void testBuffer() throws IOException { Map sources = new HashMap(); Map sinks = new HashMap(); sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) ); sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) ); List steps = getPlatform().getFlowConnector().connect( sources, sinks, pipe ).getFlowSteps(); assertEquals( "wrong size", 1, steps.size() ); BaseFlowStep step = (BaseFlowStep) steps.get( 0 ); assertEquals( "not equal: step.sources.size()", 1, step.getSourceTaps().size() ); assertNotNull( "null: step.groupBy", step.getGroup() ); assertNotNull( "null: step.sink", step.getSink() ); int mapDist = ElementGraphs.shortestDistance( step.getElementGraph(), (FlowElement) step.getSourceTaps().iterator().next(), step.getGroup() ); assertEquals( "not equal: mapDist", 1, mapDist ); int reduceDist = ElementGraphs.shortestDistance( step.getElementGraph(), step.getGroup(), step.getSink() ); assertEquals( "not equal: reduceDist", 2, reduceDist ); }
@Test public void testSideFileCollector() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "sidefile/direct" ), SinkMode.REPLACE ); Tap sideFile = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "sidefile/indirect" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( sideFile, new Fields( "next" ), 2, true, true, "next" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 23, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "1\tnull\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\ta\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\tb\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\tc\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\tnull\tnext" ) ) ); results = asList( flow, sideFile ); assertEquals( 13, results.size() ); assertTrue( results.contains( new Tuple( "1\ta" ) ) ); assertTrue( results.contains( new Tuple( "1\tb" ) ) ); assertTrue( results.contains( new Tuple( "1\tc" ) ) ); }
@Test public void testBufferFail2() throws IOException { Map sources = new HashMap(); Map sinks = new HashMap(); sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) ); sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) ); try { getPlatform().getFlowConnector().connect( sources, sinks, pipe ); fail( "did not throw planner exception" ); } catch( Exception exception ) { // ignore // exception.printStackTrace(); } }
@Test public void testSideFileCollector() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "sidefile/direct" ), SinkMode.REPLACE ); Tap sideFile = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "sidefile/indirect" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( sideFile, new Fields( "next" ), 2, true, true, "next" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 23, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "1\tnull\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\ta\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\tb\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\tc\tnext" ) ) ); assertTrue( results.contains( new Tuple( "1\tnull\tnext" ) ) ); results = asList( flow, sideFile ); assertEquals( 13, results.size() ); assertTrue( results.contains( new Tuple( "1\ta" ) ) ); assertTrue( results.contains( new Tuple( "1\tb" ) ) ); assertTrue( results.contains( new Tuple( "1\tc" ) ) ); }
@Test public void testBufferFail() throws IOException { Map sources = new HashMap(); Map sinks = new HashMap(); sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) ); sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) ); try { getPlatform().getFlowConnector().connect( sources, sinks, pipe ); fail( "did not throw planner exception" ); } catch( Exception exception ) { // ignore // exception.printStackTrace(); } }
@Test public void testBufferFail() throws IOException { Map sources = new HashMap(); Map sinks = new HashMap(); sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) ); sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) ); try { getPlatform().getFlowConnector().connect( sources, sinks, pipe ); fail( "did not throw planner exception" ); } catch( Exception exception ) { // ignore // exception.printStackTrace(); } }
@Test public void testBufferFail2() throws IOException { Map sources = new HashMap(); Map sinks = new HashMap(); sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) ); sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) ); try { getPlatform().getFlowConnector().connect( sources, sinks, pipe ); fail( "did not throw planner exception" ); } catch( Exception exception ) { // ignore // exception.printStackTrace(); } }
/** * tests wildcard fields are properly resolving. * * @throws Exception */ @Test public void testIdentityBuffer() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "identity" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, Fields.VALUES, new TestBuffer( Fields.ARGS ), Fields.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 13 ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "1\ta" ) ) ); assertTrue( results.contains( new Tuple( "1\tb" ) ) ); assertTrue( results.contains( new Tuple( "1\tc" ) ) ); }
/** * tests wildcard fields are properly resolving. * * @throws Exception */ @Test public void testIdentityBuffer() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "identity" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, Fields.VALUES, new TestBuffer( Fields.ARGS ), Fields.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 13 ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "1\ta" ) ) ); assertTrue( results.contains( new Tuple( "1\tb" ) ) ); assertTrue( results.contains( new Tuple( "1\tc" ) ) ); }
@Test public void testSimpleBuffer() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( new Fields( "next" ), 2, true, true, "next" ) ); pipe = new Each( pipe, new Insert( new Fields( "final" ), "final" ), Fields.ALL ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 23, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "1\tnull\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\ta\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\tb\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\tc\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\tnull\tnext\tfinal" ) ) ); }
@Test public void testSimpleBuffer3() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple3" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( new Fields( "new" ), new Tuple( "new" ) ), new Fields( "new", "lower", "upper" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "new\ta\tA" ) ) ); assertTrue( results.contains( new Tuple( "new\tb\tB" ) ) ); assertTrue( results.contains( new Tuple( "new\tc\tC" ) ) ); }
@Test public void testSimpleBuffer3() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple3" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( new Fields( "new" ), new Tuple( "new" ) ), new Fields( "new", "lower", "upper" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "new\ta\tA" ) ) ); assertTrue( results.contains( new Tuple( "new\tb\tB" ) ) ); assertTrue( results.contains( new Tuple( "new\tc\tC" ) ) ); }
@Test public void testSimpleBuffer() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( new Fields( "next" ), 2, true, true, "next" ) ); pipe = new Each( pipe, new Insert( new Fields( "final" ), "final" ), Fields.ALL ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 23, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "1\tnull\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\ta\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\tb\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\tc\tnext\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "1\tnull\tnext\tfinal" ) ) ); }
@Test public void testSimpleBuffer2() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple2" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new Fields( "lower" ), new TestBuffer( new Fields( "next" ), 1, true, "next" ), Fields.RESULTS ); pipe = new Each( pipe, new Insert( new Fields( "final" ), "final" ), Fields.ALL ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 18, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "next\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "next\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "next\tfinal" ) ) ); }
@Test public void testSimpleBuffer2() throws Exception { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple2" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new Fields( "lower" ), new TestBuffer( new Fields( "next" ), 1, true, "next" ), Fields.RESULTS ); pipe = new Each( pipe, new Insert( new Fields( "final" ), "final" ), Fields.ALL ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 18, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "next\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "next\tfinal" ) ) ); assertTrue( results.contains( new Tuple( "next\tfinal" ) ) ); }