public static FlowProcess undelegate( FlowProcess flowProcess ) { if( flowProcess instanceof FlowProcessWrapper ) return ( (FlowProcessWrapper) flowProcess ).getDelegate(); return flowProcess; }
public HadoopSinkStage( FlowProcess flowProcess, Tap sink ) { super( flowProcess, sink ); hadoopFlowProcess = (HadoopFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); }
public static synchronized CompressionCodec getCodec( FlowProcess<? extends Configuration> flowProcess, String defaultCodecs ) { Class<? extends CompressionCodec> codecClass = getCodecClass( flowProcess, defaultCodecs, CompressionCodec.class ); if( codecClass == null ) return null; if( flowProcess instanceof FlowProcessWrapper ) flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); return ReflectionUtils.newInstance( codecClass, flowProcess.getConfig() ); }
public HadoopSinkStage( FlowProcess flowProcess, Tap sink ) { super( flowProcess, sink ); hadoopFlowProcess = (HadoopFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); }
public static synchronized CompressionCodec getCodec( FlowProcess<? extends Configuration> flowProcess, String defaultCodecs ) { Class<? extends CompressionCodec> codecClass = getCodecClass( flowProcess, defaultCodecs, CompressionCodec.class ); if( codecClass == null ) return null; if( flowProcess instanceof FlowProcessWrapper ) flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); return ReflectionUtils.newInstance( codecClass, flowProcess.getConfig() ); }
if( ( (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getContext().canCommit() ) break;
@Override public void sinkPrepare( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"sink-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); super.sinkPrepare( flowProcess, sinkCall ); } }
@Override public void prepare() { LOG.info( "calling {}#start() on: {}", logicalInput.getClass().getSimpleName(), getSource() ); logicalInput.start(); Hadoop2TezFlowProcess tezFlowProcess = (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); TezConfiguration configuration = tezFlowProcess.getConfiguration(); try { reader = (MRReader) logicalInput.getReader(); } catch( IOException exception ) { throw new CascadeException( "unable to get reader", exception ); } // set the cascading.source.path property for the current split // if a TezGroupedSplit, currently won't set TezUtil.setSourcePathForSplit( logicalInput, reader, configuration ); }
@Override public void prepare( FlowProcess flowProcess, OperationCall operationCall ) { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"pipe-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); if( supportsNodeConfig && !"node-replace".equals( flowProcess.getProperty( "default-node" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); }
if( ( (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getContext().canCommit() ) break;
@Override public void sourcePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"source-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); super.sourcePrepare( flowProcess, sourceCall ); }
@Override public void prepare() { LOG.info( "calling {}#start() on: {}", logicalInput.getClass().getSimpleName(), getSource() ); logicalInput.start(); Hadoop2TezFlowProcess tezFlowProcess = (Hadoop2TezFlowProcess) FlowProcessWrapper.undelegate( flowProcess ); TezConfiguration configuration = tezFlowProcess.getConfiguration(); try { reader = (MRReader) logicalInput.getReader(); } catch( IOException exception ) { throw new CascadeException( "unable to get reader", exception ); } // set the cascading.source.path property for the current split // if a TezGroupedSplit, currently won't set TezUtil.setSourcePathForSplit( logicalInput, reader, configuration ); }
@Override public void prepare( FlowProcess flowProcess, OperationCall operationCall ) { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"pipe-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); if( supportsNodeConfig && !"node-replace".equals( flowProcess.getProperty( "default-node" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); }
@Override protected TupleEntrySchemeCollector createTupleEntrySchemeCollector( FlowProcess<? extends Properties> flowProcess, Tap parent, String path, long sequence ) throws IOException { if( sequence != -1 && flowProcess.getConfig() != null ) ( (LocalFlowProcess) FlowProcessWrapper.undelegate( flowProcess ) ).getConfig().setProperty( PART_NUM_PROPERTY, Long.toString( sequence ) ); if( parent instanceof TapWith ) return (TupleEntrySchemeCollector) ( (TapWith) parent ) .withChildIdentifier( path ) .withSinkMode( SinkMode.UPDATE ) .asTap().openForWrite( flowProcess ); TapFileOutputStream output = new TapFileOutputStream( parent, path, true ); // always append return new TupleEntrySchemeCollector<Properties, OutputStream>( flowProcess, parent, output ); }
@Override public void sinkPrepare( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"sink-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); super.sinkPrepare( flowProcess, sinkCall ); } }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"source-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); if( supportsNodeConfig && !"node-replace".equals( flowProcess.getProperty( "default-node" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); super.sourcePrepare( flowProcess, sourceCall ); }
@Override public void sinkPrepare( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"sink-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); super.sinkPrepare( flowProcess, sinkCall ); } }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( !( flowProcess instanceof FlowProcessWrapper ) ) throw new RuntimeException( "not a flow process wrapper" ); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"source-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); if( supportsNodeConfig && !"node-replace".equals( flowProcess.getProperty( "default-node" ) ) ) throw new RuntimeException( "not replaced value" ); flowProcess = ( (FlowProcessWrapper) flowProcess ).getDelegate(); if( !"process-default".equals( flowProcess.getProperty( "default" ) ) ) throw new RuntimeException( "not default value" ); if( !"process-replace".equals( flowProcess.getProperty( "replace" ) ) ) throw new RuntimeException( "not replaced value" ); super.sourcePrepare( flowProcess, sourceCall ); }