@Override public void sourceCleanup(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourceCleanup(flowProcess, sourceCall); sourceCall.setContext(null); }
@Override public void sourcePrepare(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { super.sourcePrepare(flowProcess, sourceCall); Object[] context = new Object[SRC_CTX_SIZE]; Settings settings = HadoopSettingsManager.loadFrom(flowProcess.getConfigCopy()).merge(props); context[SRC_CTX_ALIASES] = CascadingUtils.alias(settings); context[SRC_CTX_OUTPUT_JSON] = settings.getOutputAsJson(); sourceCall.setContext(context); }
@Override public void sourcePrepare(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourcePrepare(flowProcess, sourceCall); Object[] context = new Object[SRC_CTX_SIZE]; context[SRC_CTX_KEY] = sourceCall.getInput().createKey(); context[SRC_CTX_VALUE] = sourceCall.getInput().createValue(); // as the tuple _might_ vary (some objects might be missing), we use a map rather then a collection Settings settings = loadSettings(flowProcess.getConfigCopy(), true); context[SRC_CTX_ALIASES] = CascadingUtils.alias(settings); context[SRC_CTX_OUTPUT_JSON] = settings.getOutputAsJson(); sourceCall.setContext(context); }
@Override public void sourceCleanup(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { // in case of a source there's no local client so do all reporting here report(sourceCall.getInput().stats(), flowProcess); report(sourceCall.getInput().repository().stats(), flowProcess); sourceCall.getInput().close(); sourceCall.setContext(null); // used for consistency cleanupClient(flowProcess); }
@Override public void sourceCleanup( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( null ); }
@Override public void sourceCleanup( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { sourceCall.setContext( null ); }
@Override public void sourceCleanup( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( null ); }
@Override public void sourceCleanup( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { sourceCall.setContext( null ); }
@Override public void sourceCleanup(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) { sourceCall.setContext(null); }
@Override public void sourceCleanup(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourceCleanup(flowProcess, sourceCall); sourceCall.setContext(null); }
@Override public void sourcePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( createInput( sourceCall.getInput() ) ); }
@Override public void sourceRePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( createInput( sourceCall.getInput() ) ); }
@Override public void sourceRePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( createInput( sourceCall.getInput() ) ); }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { Object[] pair = new Object[]{ sourceCall.getInput().createKey(), sourceCall.getInput().createValue() }; sourceCall.setContext( pair ); }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { Object[] pair = new Object[]{ sourceCall.getInput().createKey(), sourceCall.getInput().createValue() }; sourceCall.setContext( pair ); }
@Override public void sourcePrepare(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) { Object[] pair = new Object[]{sourceCall.getInput().createKey(), sourceCall.getInput().createValue()}; sourceCall.setContext(pair); }
@Override public void sourcePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( createInput( sourceCall.getInput() ) ); sourceCall.getIncomingEntry().setTuple( TupleViews.createObjectArray() ); }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { if( sourceCall.getContext() == null ) sourceCall.setContext( new Object[ 3 ] ); sourceCall.getContext()[ 0 ] = sourceCall.getInput().createKey(); sourceCall.getContext()[ 1 ] = sourceCall.getInput().createValue(); sourceCall.getContext()[ 2 ] = Charset.forName( charsetName ); }
/** * This sets up the state between succesive calls to source */ @Override public void sourcePrepare(FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall) { //Hadoop sets a key value pair: sourceCall.setContext(new Object[2]); sourceCall.getContext()[0] = sourceCall.getInput().createKey(); sourceCall.getContext()[1] = sourceCall.getInput().createValue(); } }
@Override public void sourceCleanup(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { // in case of a source there's no local client so do all reporting here report(sourceCall.getInput().stats(), flowProcess); report(sourceCall.getInput().repository().stats(), flowProcess); sourceCall.getInput().close(); sourceCall.setContext(null); // used for consistency cleanupClient(flowProcess); }