private void testFilteredPartitionTapFields( Fields partitionFields, Fields argumentSelector ) throws Exception { Tap tap = getPlatform().getTextFile( "dummy" ); Partition partition = new DelimitedPartition( partitionFields ); PartitionTap partitionTap = (PartitionTap) getPlatform().getPartitionTap( tap, partition, 1 ); partitionTap.addSourcePartitionFilter( argumentSelector, new TrueFilter() ); }
private void testFilteredPartitionTapFields( Fields partitionFields, Fields argumentSelector ) throws Exception { Tap tap = getPlatform().getTextFile( "dummy" ); Partition partition = new DelimitedPartition( partitionFields ); PartitionTap partitionTap = (PartitionTap) getPlatform().getPartitionTap( tap, partition, 1 ); partitionTap.addSourcePartitionFilter( argumentSelector, new TrueFilter() ); }
@Test public void testFilteredPartitionTap_Typical() throws Exception { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getDelimitedFile( new Fields( "number", "lower" ), " ", inputFileLower ); Tap delimitedFile = getPlatform().getDelimitedFile( new Fields( "lower" ), "+", getOutputPath( "/filteredpartition/partitioned" ), SinkMode.REPLACE ); Partition partition = new DelimitedPartition( new Fields( "number" ) ); PartitionTap partitionTap = (PartitionTap) getPlatform().getPartitionTap( delimitedFile, partition, 1 ); Flow firstFlow = getPlatform().getFlowConnector().connect( source, partitionTap, new Pipe( "partition" ) ); firstFlow.complete(); partitionTap = (PartitionTap) getPlatform().getPartitionTap( delimitedFile, partition, 1 ); partitionTap.addSourcePartitionFilter( new Fields( "number" ), new PartitionFilter( Arrays.asList( "2", "4" ) ) ); Tap sink = getPlatform().getDelimitedFile( new Fields( "number", "lower" ), "+", getOutputPath( "/filteredpartition/final" ), SinkMode.REPLACE ); Flow secondFlow = getPlatform().getFlowConnector().connect( partitionTap, sink, new Pipe( "copy" ) ); secondFlow.complete(); List<Tuple> values = getSinkAsList( secondFlow ); assertEquals( 3, values.size() ); assertTrue( values.contains( new Tuple( "1", "a" ) ) ); assertTrue( values.contains( new Tuple( "3", "c" ) ) ); assertTrue( values.contains( new Tuple( "5", "e" ) ) ); }
@Test public void testFilteredPartitionTap_Typical() throws Exception { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getDelimitedFile( new Fields( "number", "lower" ), " ", inputFileLower ); Tap delimitedFile = getPlatform().getDelimitedFile( new Fields( "lower" ), "+", getOutputPath( "/filteredpartition/partitioned" ), SinkMode.REPLACE ); Partition partition = new DelimitedPartition( new Fields( "number" ) ); PartitionTap partitionTap = (PartitionTap) getPlatform().getPartitionTap( delimitedFile, partition, 1 ); Flow firstFlow = getPlatform().getFlowConnector().connect( source, partitionTap, new Pipe( "partition" ) ); firstFlow.complete(); partitionTap = (PartitionTap) getPlatform().getPartitionTap( delimitedFile, partition, 1 ); partitionTap.addSourcePartitionFilter( new Fields( "number" ), new PartitionFilter( Arrays.asList( "2", "4" ) ) ); Tap sink = getPlatform().getDelimitedFile( new Fields( "number", "lower" ), "+", getOutputPath( "/filteredpartition/final" ), SinkMode.REPLACE ); Flow secondFlow = getPlatform().getFlowConnector().connect( partitionTap, sink, new Pipe( "copy" ) ); secondFlow.complete(); List<Tuple> values = getSinkAsList( secondFlow ); assertEquals( 3, values.size() ); assertTrue( values.contains( new Tuple( "1", "a" ) ) ); assertTrue( values.contains( new Tuple( "3", "c" ) ) ); assertTrue( values.contains( new Tuple( "5", "e" ) ) ); }