@Override public FlowProcess copyWith( JobConf jobConf ) { return new HadoopFlowProcess( this, jobConf ); }
@Override public Reporter getReporter() { return hadoopProcess.getReporter(); } };
@Override public int getNumProcessSlices() { if( isMapper() ) return getCurrentNumMappers(); else return getCurrentNumReducers(); }
public HadoopFlowProcess( HadoopFlowProcess flowProcess, JobConf jobConf ) { super( flowProcess ); this.jobConf = jobConf; this.isMapper = flowProcess.isMapper(); this.reporter = flowProcess.getReporter(); }
@Override public boolean commitResource(JobConf conf) throws java.io.IOException { TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this); System.out.println(""); System.out.println(""); System.out.println("RESULTS"); System.out.println("-----------------------"); while (it.hasNext()) { System.out.println(it.next().getTuple()); } System.out.println("-----------------------"); it.close(); return true; } }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { JobConf jobConf = HadoopUtil.copyJobConf( getJobConf() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); String partname; if( jobConf.getBoolean( "mapred.task.is.map", true ) ) partname = String.format( "-m-%05d-", stepNum ); else partname = String.format( "-r-%05d-", stepNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new HadoopFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
/** * Method getCurrentTaskNum returns the task number of this task. Task 0 is the first task. * * @return int */ @Override public int getCurrentSliceNum() { return getJobConf().getInt( "mapred.task.partition", 0 ); }
protected OutputCollector createOutputCollector() { return ( (HadoopFlowProcess) flowProcess ).getOutputCollector(); } }
public HadoopMapStreamGraph( HadoopFlowProcess flowProcess, FlowNode node, Tap source ) { super( flowProcess, node, source ); this.source = source; buildGraph(); setTraps(); setScopes(); printGraph( node.getID(), "map", flowProcess.getCurrentSliceNum() ); bind(); printBoundGraph( node.getID(), "map", flowProcess.getCurrentSliceNum() ); }
@Override public boolean commitResource(JobConf conf) throws java.io.IOException { TupleEntryIterator it = new HadoopFlowProcess(conf).openTapForRead(this); boolean first_time = true; while (it.hasNext()) { TupleEntry tuple = it.next(); results.add(tuple.getTupleCopy()); if (first_time) { fields = tuple.getFields(); first_time = false; } } it.close(); return true; } }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { JobConf jobConf = HadoopUtil.copyJobConf( getJobConf() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); String partname; if( jobConf.getBoolean( "mapred.task.is.map", true ) ) partname = String.format( "-m-%05d-", stepNum ); else partname = String.format( "-r-%05d-", stepNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new HadoopFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
public HadoopFlowProcess( HadoopFlowProcess flowProcess, JobConf jobConf ) { super( flowProcess ); this.jobConf = jobConf; this.isMapper = flowProcess.isMapper(); this.reporter = flowProcess.getReporter(); }
/** * Method getCurrentTaskNum returns the task number of this task. Task 0 is the first task. * * @return int */ @Override public int getCurrentSliceNum() { return getJobConf().getInt( "mapred.task.partition", 0 ); }
@Override protected Object getOutput() { return hadoopFlowProcess.getOutputCollector(); } }
public HadoopReduceStreamGraph( HadoopFlowProcess flowProcess, FlowNode node, FlowElement sourceElement ) { super( flowProcess, node, sourceElement ); buildGraph(); setTraps(); setScopes(); printGraph( node.getID(), "reduce", flowProcess.getCurrentSliceNum() ); bind(); printBoundGraph( node.getID(), "reduce", flowProcess.getCurrentSliceNum() ); }
@Override public FlowProcess copyWith( JobConf jobConf ) { return new HadoopFlowProcess( this, jobConf ); }
@Override public boolean isCounterStatusInitialized() { return getReporter() != null; }
Tap progressEdgesSink = runUserIdNormalizationIteration(i); if(!new HadoopFlowProcess(new JobConf()) .openTapForRead(progressEdgesSink) .hasNext()) { break;
@Override public int getNumProcessSlices() { if( isMapper() ) return getCurrentNumMappers(); else return getCurrentNumReducers(); }
public int getCurrentNumMappers() { return getJobConf().getNumMapTasks(); }