@Override protected boolean areCountersAvailable( FlowNodeStats runningJob ) { return !HadoopUtil.isLocal( this.configuration ); }
@Override protected boolean isRemoteExecution() { return !HadoopUtil.isLocal( jobConfiguration ); }
@Override protected boolean areCountersAvailable( FlowNodeStats runningJob ) { return !HadoopUtil.isLocal( this.configuration ); }
@Override protected boolean isRemoteExecution() { return !HadoopUtil.isLocal( jobConfiguration ); }
public boolean isHadoopLocalMode( JobConf conf ) { return HadoopUtil.isLocal( conf ); }
public boolean isHadoopLocalMode( JobConf conf ) { return HadoopUtil.isLocal( conf ); }
@Override public boolean stepsAreLocal() { return HadoopUtil.isLocal( getConfig() ); }
@Override public boolean stepsAreLocal() { return HadoopUtil.isLocal( getConfig() ); }
@Override public boolean stepsAreLocal() { return HadoopUtil.isLocal( getConfig() ); }
@Override public boolean stepsAreLocal() { return HadoopUtil.isLocal( getConfig() ); }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
/** * Report the error to HadoopFlowStepJob if we are running in Hadoops local mode. * * @param throwable The throwable that was thrown. */ private void reportIfLocal( Throwable throwable ) { if( HadoopUtil.isLocal( currentProcess.getJobConf() ) ) HadoopFlowStepJob.reportLocalError( throwable ); } }
private void setLocalMode( Configuration parent, JobConf current, Tap tap ) { // force step to local mode if( !HadoopUtil.isLocal( current ) ) return; if( tap != null ) logInfo( "tap forcing step to tez local mode: " + tap.getIdentifier() ); HadoopUtil.setLocal( parent ); }
private void setLocalMode( Configuration parent, JobConf current, Tap tap ) { // force step to local mode if( !HadoopUtil.isLocal( current ) ) return; if( tap != null ) logInfo( "tap forcing step to tez local mode: " + tap.getIdentifier() ); HadoopUtil.setLocal( parent ); }
String createStepStagingPath() { String result = ""; if( HadoopUtil.isLocal( jobConfiguration ) ) result = jobConfiguration.get( "hadoop.tmp.dir" ) + Path.SEPARATOR; String flowStagingPath = ( (Hadoop2TezFlow) flowStep.getFlow() ).getFlowStagingPath(); return result + flowStagingPath + Path.SEPARATOR + flowStep.getID(); }
String createStepStagingPath() { String result = ""; if( HadoopUtil.isLocal( jobConfiguration ) ) result = jobConfiguration.get( "hadoop.tmp.dir" ) + Path.SEPARATOR; String flowStagingPath = ( (Hadoop2TezFlow) flowStep.getFlow() ).getFlowStagingPath(); return result + flowStagingPath + Path.SEPARATOR + flowStep.getID(); }
@Test public void testLocalModeSink() throws Exception { Tap source = new Hfs( new TextLine(), "input/path" ); Tap sink = new Lfs( new TextLine(), "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "wrong size", 1, steps.size() ); FlowStep step = steps.get( 0 ); boolean isLocal = HadoopUtil.isLocal( (Configuration) step.getConfig() ); assertTrue( "is not local", isLocal ); }
@Test public void testLocalModeSource() throws Exception { Tap source = new Lfs( new TextLine(), "input/path" ); Tap sink = new Hfs( new TextLine(), "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "wrong size", 1, steps.size() ); FlowStep step = steps.get( 0 ); boolean isLocal = HadoopUtil.isLocal( (Configuration) step.getConfig() ); assertTrue( "is not local", isLocal ); }