/** * Method getCurrentTaskNum returns the task number of this task. Task 0 is the first task. * * @return int */ @Override public int getCurrentSliceNum() { return getJobConf().getInt( "mapred.task.partition", 0 ); }
public int getCurrentNumReducers() { return getJobConf().getNumReduceTasks(); }
public int getCurrentNumMappers() { return getJobConf().getNumMapTasks(); }
/** * Method getCurrentTaskNum returns the task number of this task. Task 0 is the first task. * * @return int */ @Override public int getCurrentSliceNum() { return getJobConf().getInt( "mapred.task.partition", 0 ); }
public int getCurrentNumMappers() { return getJobConf().getNumMapTasks(); }
public int getCurrentNumReducers() { return getJobConf().getNumReduceTasks(); }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { JobConf jobConf = HadoopUtil.copyJobConf( getJobConf() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); String partname; if( jobConf.getBoolean( "mapred.task.is.map", true ) ) partname = String.format( "-m-%05d-", stepNum ); else partname = String.format( "-r-%05d-", stepNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new HadoopFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { JobConf jobConf = HadoopUtil.copyJobConf( getJobConf() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); String partname; if( jobConf.getBoolean( "mapred.task.is.map", true ) ) partname = String.format( "-m-%05d-", stepNum ); else partname = String.format( "-r-%05d-", stepNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new HadoopFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
protected void buildGraph() { streamedHead = handleHead( this.source, flowProcess ); Set<Tap> tributaries = ElementGraphs.findSources( elementGraph, Tap.class ); tributaries.remove( this.source ); // we cannot stream and accumulate the same source // accumulated paths for( Object source : tributaries ) { final HadoopFlowProcess hadoopProcess = (HadoopFlowProcess) flowProcess; JobConf conf = hadoopProcess.getJobConf(); // allows client side config to be used cluster side String property = conf.getRaw( "cascading.node.accumulated.source.conf." + Tap.id( (Tap) source ) ); if( property == null ) throw new IllegalStateException( "accumulated source conf property missing for: " + ( (Tap) source ).getIdentifier() ); conf = getSourceConf( hadoopProcess, conf, property ); // the reporter isn't provided until after the #run method is called flowProcess = new HadoopFlowProcess( hadoopProcess, conf ) { @Override public Reporter getReporter() { return hadoopProcess.getReporter(); } }; handleHead( (Tap) source, flowProcess ); } }
protected void buildGraph() { streamedHead = handleHead( this.source, flowProcess ); Set<Tap> tributaries = ElementGraphs.findSources( elementGraph, Tap.class ); tributaries.remove( this.source ); // we cannot stream and accumulate the same source // accumulated paths for( Object source : tributaries ) { final HadoopFlowProcess hadoopProcess = (HadoopFlowProcess) flowProcess; JobConf conf = hadoopProcess.getJobConf(); // allows client side config to be used cluster side String property = conf.getRaw( "cascading.node.accumulated.source.conf." + Tap.id( (Tap) source ) ); if( property == null ) throw new IllegalStateException( "accumulated source conf property missing for: " + ( (Tap) source ).getIdentifier() ); conf = getSourceConf( hadoopProcess, conf, property ); // the reporter isn't provided until after the #run method is called flowProcess = new HadoopFlowProcess( hadoopProcess, conf ) { @Override public Reporter getReporter() { return hadoopProcess.getReporter(); } }; handleHead( (Tap) source, flowProcess ); } }