/** * Method getFullIdentifier returns a fully qualified resource identifier. * * @param conf of type Config * @return String */ public String getFullIdentifier( Config conf ) { return getIdentifier(); }
@Override public String getIdentifier() { return original.getIdentifier(); }
@Override public String getIdentifier() { return original.getIdentifier(); }
@Override public String getIdentifier() { return parent.getIdentifier(); }
@Override public Tap getSinkWith( String identifier ) { if( Util.isEmpty( identifier ) ) return null; for( Tap tap : sinks.keySet() ) { if( identifier.equalsIgnoreCase( tap.getIdentifier() ) ) return tap; } return null; }
@Override public Tap getSourceWith( String identifier ) { if( Util.isEmpty( identifier ) ) return null; for( Tap tap : sources.keySet() ) { if( identifier.equalsIgnoreCase( tap.getIdentifier() ) ) return tap; } return null; }
/** * Called from flow step to remove temp dirs * * @param conf * @throws IOException */ public static void cleanupTapMetaData( Configuration conf, Tap tap ) throws IOException { cleanTempPath( conf, new Path( tap.getIdentifier() ) ); }
/** * Called from flow step to remove temp dirs * * @param conf * @throws IOException */ public static void cleanupTapMetaData( Configuration conf, Tap tap ) throws IOException { cleanTempPath( conf, new Path( tap.getIdentifier() ) ); }
@Override public int hashCode() { int result = getIdentifier() != null ? getIdentifier().hashCode() : 0; result = 31 * result + ( getScheme() != null ? getScheme().hashCode() : 0 ); return result; }
@Override public String toString() { if( getIdentifier() != null ) return getClass().getSimpleName() + "[\"" + getScheme() + "\"]" + "[\"" + Util.sanitizeUrl( getIdentifier() ) + "\"]"; // sanitize else return getClass().getSimpleName() + "[\"" + getScheme() + "\"]" + "[not initialized]"; } }
private Map<String, Tap> makeTapMap( Object resource ) { Collection paths = makeCollection( resource ); Map<String, Tap> taps = new HashMap<String, Tap>(); for( Object path : paths ) { if( path instanceof Tap && ( (Tap) path ).getIdentifier() != null ) taps.put( ( (Tap) path ).getIdentifier(), (Tap) path ); else taps.put( path.toString(), new ProcessTap( new NullScheme(), path.toString() ) ); } return taps; }
public String makeFlowStepName( FlowStep flowStep, int numSteps, int stepNum ) { Tap sink = Util.getFirst( flowStep.getSinkTaps() ); stepNum++; // number more sensical (5/5) if( sink == null || sink.isTemporary() ) return String.format( "(%d/%d)", stepNum, numSteps ); String identifier = sink.getIdentifier(); if( Util.isEmpty( identifier ) ) return String.format( "(%d/%d)", stepNum, numSteps ); if( identifier.length() > 25 ) identifier = String.format( "...%25s", identifier.substring( identifier.length() - 25 ) ); return String.format( "(%d/%d) %s", stepNum, numSteps, identifier ); }
public HadoopTupleEntrySchemeCollector( FlowProcess<? extends Configuration> flowProcess, Tap<Configuration, RecordReader, OutputCollector> tap ) throws IOException { super( flowProcess, tap, tap.getScheme(), makeCollector( flowProcess, tap, null ), tap.getIdentifier() ); }
@Override public void finalizeGlobal(int parallelism) throws IOException { org.apache.hadoop.conf.Configuration config = HadoopUtil.copyConfiguration(this.config); Tap tap = this.flowNode.getSinkTaps().iterator().next(); config.setBoolean(HadoopUtil.CASCADING_FLOW_EXECUTING, false); HadoopUtil.setOutputPath(config, new Path(tap.getIdentifier())); Hadoop18TapUtil.cleanupJob( config ); } }
private void initialize() throws IOException { tap.sinkConfInit(hadoopFlowProcess, conf); OutputFormat outputFormat = conf.getOutputFormat(); LOG.info("Output format class is: " + outputFormat.getClass().toString()); writer = outputFormat.getRecordWriter(null, conf, tap.getIdentifier(), Reporter.NULL); sinkCall.setOutput(this); }
private void setLocalMode( Configuration parent, JobConf current, Tap tap ) { // force step to local mode if( !HadoopUtil.isLocal( current ) ) return; if( tap != null ) logInfo( "tap forcing step to tez local mode: " + tap.getIdentifier() ); HadoopUtil.setLocal( parent ); }
@Override public boolean deleteResource( Properties conf ) throws IOException { String[] childIdentifiers = ( (FileTap) parent ).getChildIdentifiers( conf, Integer.MAX_VALUE, false ); if( childIdentifiers.length == 0 ) return deleteParent( conf ); DirTap.deleteChildren( Paths.get( parent.getIdentifier() ), childIdentifiers ); return deleteParent( conf ); }
private void setLocalMode( Configuration parent, JobConf current, Tap tap ) { // force step to local mode if( !HadoopUtil.isLocal( current ) ) return; if( tap != null ) logInfo( "tap forcing step to tez local mode: " + tap.getIdentifier() ); HadoopUtil.setLocal( parent ); }
private void assertHeaders( Tap output, Flow flow ) throws IOException { TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getTextFile( new Fields( "line" ), output.getIdentifier() ) ); assertEquals( iterator.next().getObject( 0 ), "first,second,third,fourth,fifth" ); iterator.close(); }
private void assertHeaders( Tap output, Flow flow ) throws IOException { TupleEntryIterator iterator = flow.openTapForRead( getPlatform().getTextFile( new Fields( "line" ), output.getIdentifier() ) ); assertEquals( iterator.next().getObject( 0 ), "first,second,third,fourth,fifth" ); iterator.close(); }