protected static Job getJob( RunningJob runningJob ) { if( runningJob == null ) // if null, job hasn't been submitted return null; Job job = Util.returnInstanceFieldIfExistsSafe( runningJob, "job" ); if( job == null ) { LOG.warn( "unable to get underlying org.apache.hadoop.mapreduce.Job from org.apache.hadoop.mapred.RunningJob, task level task counters will be unavailable" ); return null; } return job; }
protected static Job getJob( RunningJob runningJob ) { if( runningJob == null ) // if null, job hasn't been submitted return null; Job job = Util.returnInstanceFieldIfExistsSafe( runningJob, "job" ); if( job == null ) { LOG.warn( "unable to get underlying org.apache.hadoop.mapreduce.Job from org.apache.hadoop.mapred.RunningJob, task level task counters will be unavailable" ); return null; } return job; }
private static FrameworkClient getFrameworkClient( DAGClient dagClient ) { if( dagClient instanceof TezTimelineClient ) return ( (TezTimelineClient) dagClient ).getFrameworkClient(); return Util.returnInstanceFieldIfExistsSafe( dagClient, "frameworkClient" ); } }
private static FrameworkClient getFrameworkClient( DAGClient dagClient ) { if( dagClient instanceof TezTimelineClient ) return ( (TezTimelineClient) dagClient ).getFrameworkClient(); return Util.returnInstanceFieldIfExistsSafe( dagClient, "frameworkClient" ); } }
static void injectIdentityMap( AbstractGraph graph ) { // this overcomes jgrapht 0.9.0 using a LinkedHashMap vs an IdentityHashMap // vertex not found errors will be thrown if this fails Object specifics = Util.returnInstanceFieldIfExistsSafe( graph, "specifics" ); if( specifics == null ) { LOG.warn( "unable to get jgrapht Specifics for identity map injection, may be using an incompatible jgrapht version" ); return; } boolean success = Util.setInstanceFieldIfExistsSafe( specifics, "vertexMapDirected", new IdentityHashMap<>() ); if( !success ) LOG.warn( "unable to set IdentityHashMap on jgrapht Specifics, may be using an incompatible jgrapht version" ); } }
public static String id( FlowElement flowElement ) { if( flowElement instanceof Pipe ) return Pipe.id( (Pipe) flowElement ); if( flowElement instanceof Tap ) return Tap.id( (Tap) flowElement ); String id = Util.returnInstanceFieldIfExistsSafe( flowElement, "id" ); if( id != null ) return id; throw new IllegalArgumentException( "id not supported for: " + flowElement.getClass().getCanonicalName() ); }
}; ApplicationId appId = Util.returnInstanceFieldIfExistsSafe( dagClient, "appId" ); String dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); TezConfiguration conf = Util.returnInstanceFieldIfExistsSafe( dagClient, "conf" ); FrameworkClient frameworkClient = Util.returnInstanceFieldIfExistsSafe( dagClient, "frameworkClient" );
}; ApplicationId appId = Util.returnInstanceFieldIfExistsSafe( dagClient, "appId" ); String dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); TezConfiguration conf = Util.returnInstanceFieldIfExistsSafe( dagClient, "conf" ); FrameworkClient frameworkClient = Util.returnInstanceFieldIfExistsSafe( dagClient, "frameworkClient" );
public static void setSourcePathForSplit( MRInput input, MRReader reader, Configuration configuration ) { Path path = null; if( Util.returnInstanceFieldIfExistsSafe( input, "useNewApi" ) ) { org.apache.hadoop.mapreduce.InputSplit newInputSplit = (org.apache.hadoop.mapreduce.InputSplit) reader.getSplit(); if( newInputSplit instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit ) path = ( (org.apache.hadoop.mapreduce.lib.input.FileSplit) newInputSplit ).getPath(); } else { org.apache.hadoop.mapred.InputSplit oldInputSplit = (org.apache.hadoop.mapred.InputSplit) reader.getSplit(); if( oldInputSplit instanceof org.apache.hadoop.mapred.FileSplit ) path = ( (org.apache.hadoop.mapred.FileSplit) oldInputSplit ).getPath(); } if( path != null ) configuration.set( FileType.CASCADING_SOURCE_PATH, path.toString() ); }
public static void setSourcePathForSplit( MRInput input, MRReader reader, Configuration configuration ) { Path path = null; if( Util.returnInstanceFieldIfExistsSafe( input, "useNewApi" ) ) { org.apache.hadoop.mapreduce.InputSplit newInputSplit = (org.apache.hadoop.mapreduce.InputSplit) reader.getSplit(); if( newInputSplit instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit ) path = ( (org.apache.hadoop.mapreduce.lib.input.FileSplit) newInputSplit ).getPath(); } else { org.apache.hadoop.mapred.InputSplit oldInputSplit = (org.apache.hadoop.mapred.InputSplit) reader.getSplit(); if( oldInputSplit instanceof org.apache.hadoop.mapred.FileSplit ) path = ( (org.apache.hadoop.mapred.FileSplit) oldInputSplit ).getPath(); } if( path != null ) configuration.set( FileType.CASCADING_SOURCE_PATH, path.toString() ); }
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }