@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { Hfs[] taps = initTapsInternal( conf ); taps[ 0 ].sourceConfInitAddInputPaths( conf, new LazyIterable<Hfs, Path>( taps ) { @Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } ); taps[ 0 ].sourceConfInitComplete( process, conf ); }
protected void applySourceConfInitIdentifiers( FlowProcess<? extends Configuration> process, Configuration conf, final String... fullIdentifiers ) { sourceConfInitAddInputPaths( conf, new LazyIterable<String, Path>( fullIdentifiers ) { @Override protected Path convert( String next ) { return new Path( next ); } } ); sourceConfInitComplete( process, conf ); }
@Override public TapWith<Configuration, RecordReader, OutputCollector> withScheme( Scheme<Configuration, RecordReader, OutputCollector, ?, ?> scheme ) { return create( scheme, getPath(), getSinkMode() ); }
@Override protected void addPropertiesTo( Properties properties ) { setTemporaryDirectory( properties, temporaryDirectory ); setLocalModeScheme( properties, localModeScheme ); setUseCombinedInput( properties, useCombinedInput ); setCombinedInputMaxSize( properties, combinedInputMaxSize ); setUseCombinedInputSafeMode( properties, combinedInputSafeMode ); } }
@Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } );
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { String fullIdentifier = getFullIdentifier( conf ); applySourceConfInitIdentifiers( process, conf, fullIdentifier ); verifyNoDuplicates( conf ); }
private void testFilteredPartitionTapFields( Fields partitionFields, Fields argumentSelector ) throws Exception { Tap tap = getPlatform().getTextFile( "dummy" ); Partition partition = new DelimitedPartition( partitionFields ); PartitionTap partitionTap = (PartitionTap) getPlatform().getPartitionTap( tap, partition, 1 ); partitionTap.addSourcePartitionFilter( argumentSelector, new TrueFilter() ); }
@Override public String[] getChildIdentifiers( Configuration conf ) throws IOException { return getChildIdentifiers( conf, 1, false ); }
@Override public Tap getDistCacheTap( Hfs parent ) { return new DistCacheTap( parent ); }
@Override public boolean deleteResource( Configuration conf ) throws IOException { String fullIdentifier = getFullIdentifier( conf ); return deleteFullIdentifier( conf, fullIdentifier ); }
@Override public Tap getPartitionTap( Tap sink, Partition partition, int openThreshold ) { return new PartitionTap( (Hfs) sink, partition, openThreshold ); }
/** * Creates a new HfsProps instance. * * @return HfsProps instance */ public static HfsProps hfsProps() { return new HfsProps(); }
public void assertEqualsSize( String message, int expected, String[] actual ) { // System.out.println( message + ": " + Arrays.toString( actual ) ); if( expected == -1 ) return; assertEquals( expected, actual.length ); }
@Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } );
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { Hfs[] taps = initTapsInternal( conf ); taps[ 0 ].sourceConfInitAddInputPaths( conf, new LazyIterable<Hfs, Path>( taps ) { @Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } ); taps[ 0 ].sourceConfInitComplete( process, conf ); }
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { String fullIdentifier = getFullIdentifier( conf ); applySourceConfInitIdentifiers( process, conf, fullIdentifier ); verifyNoDuplicates( conf ); }
protected void applySourceConfInitIdentifiers( FlowProcess<? extends Configuration> process, Configuration conf, final String... fullIdentifiers ) { sourceConfInitAddInputPaths( conf, new LazyIterable<String, Path>( fullIdentifiers ) { @Override protected Path convert( String next ) { return new Path( next ); } } ); sourceConfInitComplete( process, conf ); }
@Override public String[] getChildIdentifiers( Configuration conf ) throws IOException { return getChildIdentifiers( conf, 1, false ); }
@Override public Tap getDistCacheTap( Hfs parent ) { return new DistCacheTap( parent ); }