@Override protected String getCurrentIdentifier( FlowProcess<? extends Configuration> flowProcess ) { String identifier = flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ); // set on current split if( identifier == null ) { if( flowProcess.getBooleanProperty( HfsProps.COMBINE_INPUT_FILES, false ) ) throw new TapException( "combined input format support, via '" + HfsProps.COMBINE_INPUT_FILES + "', may not be enabled for use with the PartitionTap" ); throw new TapException( "unable to retrieve the current file being processed, '" + FileType.CASCADING_SOURCE_PATH + "' is not set" ); } return new Path( identifier ).getParent().toString(); // drop part-xxxx }
@Override protected String getCurrentIdentifier( FlowProcess<? extends Configuration> flowProcess ) { String identifier = flowProcess.getStringProperty( FileType.CASCADING_SOURCE_PATH ); // set on current split if( identifier == null ) { if( flowProcess.getBooleanProperty( HfsProps.COMBINE_INPUT_FILES, false ) ) throw new TapException( "combined input format support, via '" + HfsProps.COMBINE_INPUT_FILES + "', may not be enabled for use with the PartitionTap" ); throw new TapException( "unable to retrieve the current file being processed, '" + FileType.CASCADING_SOURCE_PATH + "' is not set" ); } return new Path( identifier ).getParent().toString(); // drop part-xxxx }
@Override public TupleEntryIterator openForRead( FlowProcess<? extends Configuration> flowProcess, RecordReader input ) throws IOException { if( flowProcess.getBooleanProperty( HfsProps.COMBINE_INPUT_FILES, false ) ) return new CombinePartitionIterator( flowProcess, input ); return super.openForRead( flowProcess, input ); }
@Override public TupleEntryIterator openForRead( FlowProcess<? extends Configuration> flowProcess, RecordReader input ) throws IOException { if( flowProcess.getBooleanProperty( HfsProps.COMBINE_INPUT_FILES, false ) ) return new CombinePartitionIterator( flowProcess, input ); return super.openForRead( flowProcess, input ); }
public TrapHandler( FlowProcess flowProcess, FlowElement flowElement, Tap trap, String trapName ) { this.flowProcess = flowProcess; this.flowElement = flowElement; this.trap = trap; this.trapName = trapName; if( flowElement instanceof Traceable ) this.elementTrace = ( (Traceable) flowElement ).getTrace(); else this.elementTrace = null; this.recordElementTrace = flowProcess.getBooleanProperty( TrapProps.RECORD_ELEMENT_TRACE, this.recordElementTrace ); this.recordThrowableMessage = flowProcess.getBooleanProperty( TrapProps.RECORD_THROWABLE_MESSAGE, this.recordThrowableMessage ); this.recordThrowableStackTrace = flowProcess.getBooleanProperty( TrapProps.RECORD_THROWABLE_STACK_TRACE, this.recordThrowableStackTrace ); this.logThrowableStackTrace = flowProcess.getBooleanProperty( TrapProps.LOG_THROWABLE_STACK_TRACE, this.logThrowableStackTrace ); this.stackTraceLineDelimiter = flowProcess.getStringProperty( TrapProps.STACK_TRACE_LINE_DELIMITER, this.stackTraceLineDelimiter ); this.stackTraceTrimLine = flowProcess.getBooleanProperty( TrapProps.STACK_TRACE_LINE_TRIM, this.stackTraceTrimLine ); this.recordAnyDiagnostics = this.recordElementTrace || this.recordThrowableMessage || this.recordThrowableStackTrace; Fields fields = new Fields(); if( this.recordElementTrace ) fields = fields.append( new Fields( "element-trace" ) ); if( this.recordThrowableMessage ) fields = fields.append( new Fields( "throwable-message" ) ); if( this.recordThrowableStackTrace ) fields = fields.append( new Fields( "throwable-stacktrace" ) ); if( fields.size() != 0 ) this.diagnosticFields = fields; this.diagnosticEntry = new TupleEntry( diagnosticFields ); }