/** * Constructor HBaseScheme creates a new HBaseScheme instance using fully qualified column names * * @param keyField of type String * @param valueFields of type Fields */ public HBaseScheme(Fields keyField, Fields valueFields) { this(keyField, Fields.fields(valueFields)); }
/** * Constructor CompositeFunction creates a new CompositeFunction instance. * * @param groupingFields of type Fields * @param argumentFields of type Fields * @param functor of type Functor * @param capacity of type int */ public CompositeFunction( Fields groupingFields, Fields argumentFields, Functor functor, int capacity ) { this( groupingFields, Fields.fields( argumentFields ), new Functor[]{functor}, capacity ); }
/** * Constructor CompositeAggregator creates a new CompositeAggregator instance. * * @param argumentFields of type Fields * @param functor of type Functor * @param aggregator of type Aggregator */ protected AggregateBy( Fields argumentFields, Functor functor, Aggregator aggregator ) { this.argumentFields = Fields.fields( argumentFields ); this.functors = new Functor[]{functor}; this.aggregators = new Aggregator[]{aggregator}; }
/** * Constructor HBaseScheme creates a new HBaseScheme instance. * * @param keyFields of type Fields * @param familyName of type String * @param valueFields of type Fields */ public HBaseScheme(Fields keyFields, String familyName, Fields valueFields) { this(keyFields, new String[]{familyName}, Fields.fields(valueFields)); }
/** * Constructor Splice creates a new Splice instance. * * @param lhs of type Pipe * @param lhsGroupFields of type Fields * @param rhs of type Pipe * @param rhsGroupFields of type Fields */ protected Splice( Pipe lhs, Fields lhsGroupFields, Pipe rhs, Fields rhsGroupFields ) { this( Pipe.pipes( lhs, rhs ), Fields.fields( lhsGroupFields, rhsGroupFields ) ); }
@ConstructorProperties({"lhsArgumentsSelector", "lhsFilter", "rhsArgumentSelector", "rhsFilter"}) protected Logic( Fields lhsArgumentSelector, Filter lhsFilter, Fields rhsArgumentSelector, Filter rhsFilter ) { this( Fields.fields( lhsArgumentSelector, rhsArgumentSelector ), filters( lhsFilter, rhsFilter ) ); }
protected void initialize( Fields groupingFields, Pipe[] pipes, Fields argumentFields, Functor functor, Aggregator aggregator ) { initialize( groupingFields, pipes, Fields.fields( argumentFields ), new Functor[]{functor}, new Aggregator[]{aggregator} ); }
/** * Constructor HashJoin creates a new HashJoin instance. * * @param lhs * @param lhsJoinFields * @param rhs * @param rhsJoinFields */ @ConstructorProperties({"lhs", "lhsJoinFields", "rhs", "rhsJoinFields"}) public HashJoin( Pipe lhs, Fields lhsJoinFields, Pipe rhs, Fields rhsJoinFields ) { super( null, Pipe.pipes( lhs, rhs ), Fields.fields( lhsJoinFields, rhsJoinFields ), null, null ); }
/** * Constructor HashJoin creates a new HashJoin instance. * * @param joinName * @param lhs * @param lhsJoinFields * @param rhs * @param rhsJoinFields */ @ConstructorProperties({"joinName", "lhs", "lhsJoinFields", "rhs", "rhsJoinFields"}) public HashJoin( String joinName, Pipe lhs, Fields lhsJoinFields, Pipe rhs, Fields rhsJoinFields ) { super( joinName, Pipe.pipes( lhs, rhs ), Fields.fields( lhsJoinFields, rhsJoinFields ), null, null ); }
/** * Constructor HashJoin creates a new HashJoin instance. * * @param lhs * @param lhsJoinFields * @param rhs * @param rhsJoinFields * @param declaredFields */ @ConstructorProperties({"lhs", "lhsJoinFields", "rhs", "rhsJoinFields", "declaredFields"}) public HashJoin( Pipe lhs, Fields lhsJoinFields, Pipe rhs, Fields rhsJoinFields, Fields declaredFields ) { super( null, Pipe.pipes( lhs, rhs ), Fields.fields( lhsJoinFields, rhsJoinFields ), declaredFields, null ); }
/** * Constructor HashJoin creates a new HashJoin instance. * * @param lhs * @param lhsJoinFields * @param rhs * @param rhsJoinFields * @param joiner */ @ConstructorProperties({"lhs", "lhsJoinFields", "rhs", "rhsJoinFields", "joiner"}) public HashJoin( Pipe lhs, Fields lhsJoinFields, Pipe rhs, Fields rhsJoinFields, Joiner joiner ) { super( null, Pipe.pipes( lhs, rhs ), Fields.fields( lhsJoinFields, rhsJoinFields ), null, null, joiner ); }
/** * Constructor Splice creates a new Splice instance. * * @param lhs of type Pipe * @param lhsGroupFields of type Fields * @param rhs of type Pipe * @param rhsGroupFields of type Fields * @param declaredFields of type Fields * @param resultGroupFields of type Fields * @param joiner of type Joiner */ protected Splice( Pipe lhs, Fields lhsGroupFields, Pipe rhs, Fields rhsGroupFields, Fields declaredFields, Fields resultGroupFields, Joiner joiner ) { this( Pipe.pipes( lhs, rhs ), Fields.fields( lhsGroupFields, rhsGroupFields ), declaredFields, resultGroupFields, joiner ); }
/** * Constructor Splice creates a new Splice instance. * * @param lhs of type Pipe * @param lhsGroupFields of type Fields * @param rhs of type Pipe * @param rhsGroupFields of type Fields * @param declaredFields of type Fields * @param joiner of type CoGrouper */ protected Splice( Pipe lhs, Fields lhsGroupFields, Pipe rhs, Fields rhsGroupFields, Fields declaredFields, Joiner joiner ) { this( Pipe.pipes( lhs, rhs ), Fields.fields( lhsGroupFields, rhsGroupFields ), declaredFields, joiner ); }
/** * Constructor HashJoin creates a new HashJoin instance. * * @param joinName * @param lhs * @param lhsJoinFields * @param rhs * @param rhsJoinFields * @param declaredFields */ @ConstructorProperties({"joinName", "lhs", "lhsJoinFields", "rhs", "rhsJoinFields", "declaredFields"}) public HashJoin( String joinName, Pipe lhs, Fields lhsJoinFields, Pipe rhs, Fields rhsJoinFields, Fields declaredFields ) { super( joinName, Pipe.pipes( lhs, rhs ), Fields.fields( lhsJoinFields, rhsJoinFields ), declaredFields, null ); }
@Test public void testUnGroup() throws Exception { copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( Fields.size( 2 ), inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( 1 ), new RegexSplitter( Fields.size( 3 ) ) ); pipe = new Each( pipe, new UnGroup( Fields.size( 2 ), new Fields( 0 ), Fields.fields( new Fields( 1 ), new Fields( 2 ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testUnGroup() throws Exception { copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( Fields.size( 2 ), inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( 1 ), new RegexSplitter( Fields.size( 3 ) ) ); pipe = new Each( pipe, new UnGroup( Fields.size( 2 ), new Fields( 0 ), Fields.fields( new Fields( 1 ), new Fields( 2 ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testUnGroupAnon() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungroupedanon" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testUnGroupAnon() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungroupedanon" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }