/** * Method getID() returns the current * * @return of type String */ public String getID() { return getStringProperty( FlowStep.CASCADING_FLOW_STEP_ID ); }
/** * Method getID returns the parent {@link Flow} ID value. * * @return of type String */ public String getID() { return currentProcess.getStringProperty( Flow.CASCADING_FLOW_ID ); } }
private String getCurrentFile() { String result = flowProcess.getStringProperty( "mapreduce.map.input.file" ); if( result == null ) result = flowProcess.getStringProperty( "map.input.file" ); return result; }
private String getCurrentFile() { String result = flowProcess.getStringProperty( "mapreduce.map.input.file" ); if( result == null ) result = flowProcess.getStringProperty( "map.input.file" ); return result; }
public HadoopTupleEntrySchemeIterator( FlowProcess<? extends Configuration> flowProcess, Tap tap, Scheme scheme, CloseableIterator<RecordReader> closeableIterator ) { super( flowProcess, tap, scheme, closeableIterator, flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ) ); }
@Deprecated public HadoopTupleEntrySchemeIterator( FlowProcess<? extends Configuration> flowProcess, Scheme scheme, CloseableIterator<RecordReader> closeableIterator ) { super( flowProcess, scheme, closeableIterator, flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ) ); }
@Deprecated public HadoopTupleEntrySchemeIterator( FlowProcess<? extends Configuration> flowProcess, Scheme scheme, CloseableIterator<RecordReader> closeableIterator ) { super( flowProcess, scheme, closeableIterator, flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ) ); }
public OperatorStage( FlowProcess flowProcess, FlowElement flowElement ) { super( flowProcess, flowElement ); this.retainCollector = Boolean.parseBoolean( flowProcess.getStringProperty( RETAIN_COLLECTOR ) ); }
private LogicalOutput findLogicalOutput( Pipe element ) { String id = Pipe.id( element ); LogicalOutput logicalOutput = outputMap.get( id ); if( logicalOutput == null ) logicalOutput = outputMap.get( flowProcess.getStringProperty( "cascading.node.sink." + id ) ); if( logicalOutput == null ) throw new IllegalStateException( "could not find output for: " + element ); return logicalOutput; }
public static void logCounters( Logger logger, String message, FlowProcess flowProcess ) { String counters = flowProcess.getStringProperty( FlowRuntimeProps.LOG_COUNTERS ); if( counters == null ) return; String[] split = counters.split( "," ); for( String value : split ) { String counter[] = value.split( ":" ); logger.info( "{} {}.{}={}", message, counter[ 0 ], counter[ 1 ], flowProcess.getCounterValue( counter[ 0 ], counter[ 1 ] ) ); } }
private LogicalOutput findLogicalOutput( Pipe element ) { String id = Pipe.id( element ); LogicalOutput logicalOutput = outputMap.get( id ); if( logicalOutput == null ) logicalOutput = outputMap.get( flowProcess.getStringProperty( "cascading.node.sink." + id ) ); if( logicalOutput == null ) throw new IllegalStateException( "could not find output for: " + element ); return logicalOutput; }
private LogicalInput findLogicalInput( Pipe element ) { String id = Pipe.id( element ); LogicalInput logicalInput = inputMap.get( id ); if( logicalInput == null ) logicalInput = inputMap.get( flowProcess.getStringProperty( "cascading.node.source." + id ) ); if( logicalInput == null ) throw new IllegalStateException( "could not find input for: " + element ); return logicalInput; }
@Override protected SinkStage createSinkStage( Tap sink ) { String id = Tap.id( sink ); LogicalOutput logicalOutput = outputMap.get( id ); if( logicalOutput == null ) logicalOutput = outputMap.get( flowProcess.getStringProperty( "cascading.node.sink." + id ) ); if( logicalOutput == null ) throw new IllegalStateException( "could not find output for: " + sink ); return new TezSinkStage( flowProcess, sink, logicalOutput ); }
@Override protected SinkStage createSinkStage( Tap sink ) { String id = Tap.id( sink ); LogicalOutput logicalOutput = outputMap.get( id ); if( logicalOutput == null ) logicalOutput = outputMap.get( flowProcess.getStringProperty( "cascading.node.sink." + id ) ); if( logicalOutput == null ) throw new IllegalStateException( "could not find output for: " + sink ); return new TezSinkStage( flowProcess, sink, logicalOutput ); }
@Override protected Path[] getLocalCacheFiles( FlowProcess<? extends Configuration> flowProcess ) throws IOException { String key = CASCADING_REMOTE_RESOURCES + Tap.id( this ); String property = flowProcess.getStringProperty( key ); if( property == null ) throw new TapException( "unable to find local resources property for: " + key ); String[] split = property.split( "," ); Path[] paths = new Path[ split.length ]; for( int i = 0; i < split.length; i++ ) paths[ i ] = new Path( split[ i ] ); return paths; }
@Override protected String getCurrentIdentifier( FlowProcess<? extends Configuration> flowProcess ) { String identifier = flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ); // set on current split if( identifier == null ) { if( flowProcess.getBooleanProperty( HfsProps.COMBINE_INPUT_FILES, false ) ) throw new TapException( "combined input format support, via '" + HfsProps.COMBINE_INPUT_FILES + "', may not be enabled for use with the PartitionTap" ); throw new TapException( "unable to retrieve the current file being processed, '" + FileType.CASCADING_SOURCE_PATH + "' is not set" ); } return new Path( identifier ).getParent().toString(); // drop part-xxxx }
@Override protected String getCurrentIdentifier( FlowProcess<? extends Configuration> flowProcess ) { String identifier = flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ); // set on current split if( identifier == null ) { if( flowProcess.getBooleanProperty( HfsProps.COMBINE_INPUT_FILES, false ) ) throw new TapException( "combined input format support, via '" + HfsProps.COMBINE_INPUT_FILES + "', may not be enabled for use with the PartitionTap" ); throw new TapException( "unable to retrieve the current file being processed, '" + FileType.CASCADING_SOURCE_PATH + "' is not set" ); } return new Path( identifier ).getParent().toString(); // drop part-xxxx }
protected SourceStage createSourceStage( Tap source, FlowProcess flowProcess ) { String id = Tap.id( source ); LogicalInput logicalInput = inputMap.get( id ); if( logicalInput == null ) logicalInput = inputMap.get( flowProcess.getStringProperty( "cascading.node.source." + id ) ); if( logicalInput == null ) return new SourceStage( flowProcess, source ); return new TezSourceStage( flowProcess, source, logicalInput ); }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( flowProcess.getStringProperty( "this.is.a.dupe" ) == null ) throw new IllegalStateException( "has no dupe config value" ); super.sourcePrepare( flowProcess, sourceCall ); } }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( flowProcess.getStringProperty( "this.is.a.dupe" ) == null ) throw new IllegalStateException( "has no dupe config value" ); super.sourcePrepare( flowProcess, sourceCall ); } }