private void registerURI( Configuration conf, Path path ) { URI uri = path.toUri(); LOG.info( "adding {} to local resource configuration ", uri ); addLocalCacheFiles( conf, uri ); }
private Hfs getHfs() { return (Hfs) getOriginal(); }
Path[] cachedFiles = getLocalCacheFiles( flowProcess ); List<Tap> taps = new ArrayList<>(); if( isSimpleGlob() ) FileStatus[] statuses = fs.globStatus( getHfs().getPath() ); paths.add( getHfs().getPath() ); taps.add( new Lfs( getScheme(), path.toString() ) );
private void registerHfs( FlowProcess<? extends Configuration> process, Configuration conf, Hfs hfs ) throws IOException { if( isSimpleGlob() ) { FileSystem fs = FileSystem.get( conf ); FileStatus[] statuses = fs.globStatus( getHfs().getPath() ); if( statuses == null || statuses.length == 0 ) throw new TapException( String.format( "glob expression %s does not match any files on the filesystem", getHfs().getPath() ) ); for( FileStatus fileStatus : statuses ) registerURI( conf, fileStatus.getPath() ); } else { registerURI( conf, hfs.getPath() ); } hfs.sourceConfInitComplete( process, conf ); }
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { if( HadoopUtil.isLocal( conf ) || Tap.id( this ).equals( conf.get( "cascading.node.source" ) ) || Tap.id( this ).equals( conf.get( "cascading.step.source" ) ) ) { LOG.info( "can't use distributed cache. reading '{}' from hdfs", super.getIdentifier() ); super.sourceConfInit( process, conf ); return; } try { registerHfs( process, conf, getHfs() ); } catch( IOException exception ) { throw new TapException( exception ); } }
Path[] cachedFiles = getLocalCacheFiles( flowProcess ); List<Tap> taps = new ArrayList<>(); if( isSimpleGlob() ) FileStatus[] statuses = fs.globStatus( getHfs().getPath() ); paths.add( getHfs().getPath() ); taps.add( new Lfs( getScheme(), path.toString() ) );
private void registerHfs( FlowProcess<? extends Configuration> process, Configuration conf, Hfs hfs ) throws IOException { if( isSimpleGlob() ) { FileSystem fs = FileSystem.get( conf ); FileStatus[] statuses = fs.globStatus( getHfs().getPath() ); if( statuses == null || statuses.length == 0 ) throw new TapException( String.format( "glob expression %s does not match any files on the filesystem", getHfs().getPath() ) ); for( FileStatus fileStatus : statuses ) registerURI( conf, fileStatus.getPath() ); } else { registerURI( conf, hfs.getPath() ); } hfs.sourceConfInitComplete( process, conf ); }
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { if( HadoopUtil.isLocal( conf ) || Tap.id( this ).equals( conf.get( "cascading.node.source" ) ) || Tap.id( this ).equals( conf.get( "cascading.step.source" ) ) ) { LOG.info( "can't use distributed cache. reading '{}' from hdfs", super.getIdentifier() ); super.sourceConfInit( process, conf ); return; } try { registerHfs( process, conf, getHfs() ); } catch( IOException exception ) { throw new TapException( exception ); } }
private void registerURI( Configuration conf, Path path ) { URI uri = path.toUri(); LOG.info( "adding {} to local resource configuration ", uri ); addLocalCacheFiles( conf, uri ); }
private Hfs getHfs() { return (Hfs) getOriginal(); }