private boolean matchWholeTuple( Tuple input, Pattern pattern ) { if( patternString == null ) return true; Matcher matcher = pattern.matcher( input.toString( "\t", false ) ); LOG.debug( "pattern: {}, matches: {}", pattern, matcher.matches() ); return matcher.matches(); }
@Override public void sink( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { sinkCall.getContext().println( sinkCall.getOutgoingEntry().getTuple().toString() ); }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Text text = (Text) sinkCall.getContext()[ 0 ]; Charset charset = (Charset) sinkCall.getContext()[ 1 ]; String line = sinkCall.getOutgoingEntry().getTuple().toString(); text.set( line.getBytes( charset ) ); // it's ok to use NULL here so the collector does not write anything sinkCall.getOutput().collect( null, text ); }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Text text = (Text) sinkCall.getContext()[ 0 ]; Charset charset = (Charset) sinkCall.getContext()[ 1 ]; String line = sinkCall.getOutgoingEntry().getTuple().toString(); text.set( line.getBytes( charset ) ); // it's ok to use NULL here so the collector does not write anything sinkCall.getOutput().collect( null, text ); }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Tuple> functionCall ) { functionCall.getContext().set( 0, functionCall.getArguments().getTuple().toString( delimiter, false ) ); functionCall.getOutputCollector().add( functionCall.getContext() ); }
public static void validateLength( TupleEntryIterator iterator, int numTuples, int tupleSize, Pattern regex ) { int count = 0; while( iterator.hasNext() ) { TupleEntry tupleEntry = iterator.next(); if( tupleSize != -1 ) assertEquals( "wrong number of elements", tupleSize, tupleEntry.size() ); if( regex != null ) assertTrue( "regex: " + regex + " does not match: " + tupleEntry.getTuple().toString(), regex.matcher( tupleEntry.getTuple().toString() ).matches() ); count++; } try { iterator.close(); } catch( IOException exception ) { throw new RuntimeException( exception ); } assertEquals( "wrong number of lines", numTuples, count ); }
@Test public void testSinkUnknown() throws IOException { getPlatform().copyFromLocal( inputFileCross ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileCross ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.RESULTS ); Tap sink = getPlatform().getTabDelimitedFile( Fields.UNKNOWN, getOutputPath( "unknownsinks" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 37, null ); TupleEntryIterator iterator = flow.openSink(); String line = iterator.next().getTuple().toString(); assertTrue( "not equal: wrong values: " + line, line.matches( "[0-9]\t[a-z]\t[A-Z]" ) ); iterator.close(); }
@Test public void testSinkUnknown() throws IOException { getPlatform().copyFromLocal( inputFileCross ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileCross ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third" ), "\\s" ), Fields.RESULTS ); Tap sink = getPlatform().getTabDelimitedFile( Fields.UNKNOWN, getOutputPath( "unknownsinks" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 37, null ); TupleEntryIterator iterator = flow.openSink(); String line = iterator.next().getTuple().toString(); assertTrue( "not equal: wrong values: " + line, line.matches( "[0-9]\t[a-z]\t[A-Z]" ) ); iterator.close(); }
assertEquals( "not equal: ", resultTuple.toString(), result ); else if( resultTuple == null ) fail( "no result assertion made for:" + getName() + " with result: " + result );
assertEquals( "not equal: ", resultTuple.toString(), result ); else if( resultTuple == null ) fail( "no result assertion made for:" + getName() + " with result: " + result );