public HadoopSinkStage( FlowProcess flowProcess, Tap sink ) { super( flowProcess, sink ); hadoopFlowProcess = (HadoopFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); }
public HadoopSinkStage( FlowProcess flowProcess, Tap sink ) { super( flowProcess, sink ); hadoopFlowProcess = (HadoopFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); }
if( ( (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getContext().canCommit() ) break;
if( ( (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getContext().canCommit() ) break;
@Override public void prepare() { LOG.info( "calling {}#start() on: {}", logicalInput.getClass().getSimpleName(), getSource() ); logicalInput.start(); Hadoop2TezFlowProcess tezFlowProcess = (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); TezConfiguration configuration = tezFlowProcess.getConfiguration(); try { reader = (MRReader) logicalInput.getReader(); } catch( IOException exception ) { throw new CascadeException( "unable to get reader", exception ); } // set the cascading.source.path property for the current split // if a TezGroupedSplit, currently won't set TezUtil.setSourcePathForSplit( logicalInput, reader, configuration ); }
@Override public void prepare() { LOG.info( "calling {}#start() on: {}", logicalInput.getClass().getSimpleName(), getSource() ); logicalInput.start(); Hadoop2TezFlowProcess tezFlowProcess = (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); TezConfiguration configuration = tezFlowProcess.getConfiguration(); try { reader = (MRReader) logicalInput.getReader(); } catch( IOException exception ) { throw new CascadeException( "unable to get reader", exception ); } // set the cascading.source.path property for the current split // if a TezGroupedSplit, currently won't set TezUtil.setSourcePathForSplit( logicalInput, reader, configuration ); }
@Override protected TupleEntrySchemeCollector createTupleEntrySchemeCollector( FlowProcess<? extends Properties> flowProcess, Tap parent, String path, long sequence ) throws IOException { if( sequence != -1 && flowProcess.getConfig() != null ) ( (LocalFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getConfig().setProperty( PART_NUM_PROPERTY, Long.toString( sequence ) ); if( parent instanceof TapWith ) return (TupleEntrySchemeCollector) ( (TapWith) parent ) .withChildIdentifier( path ) .withSinkMode( SinkMode.UPDATE ) .asTap().openForWrite( flowProcess ); TapFileOutputStream output = new TapFileOutputStream( parent, path, true ); // always append return new TupleEntrySchemeCollector<Properties, OutputStream>( flowProcess, parent, output ); }