setup(job); job.fs = job.getFileSystem(job.conf); TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId); job.numMapTasks = taskSplitMetaInfo.length; job.numReduceTasks = job.conf.getInt(MRJobConfig.NUM_REDUCES, 0); checkTaskLimits(); createMapTasks(job, inputLength, taskSplitMetaInfo); createReduceTasks(job);
setup(job); job.fs = job.getFileSystem(job.conf); TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId); job.numMapTasks = taskSplitMetaInfo.length; job.numReduceTasks = job.conf.getInt(MRJobConfig.NUM_REDUCES, 0); checkTaskLimits(); createMapTasks(job, inputLength, taskSplitMetaInfo); createReduceTasks(job);
setup(job); job.fs = job.getFileSystem(job.conf); TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId); job.numMapTasks = taskSplitMetaInfo.length; job.numReduceTasks = job.conf.getInt(MRJobConfig.NUM_REDUCES, 0); checkTaskLimits(); createMapTasks(job, inputLength, taskSplitMetaInfo); createReduceTasks(job);
@Test public void testMetaInfoSizeOverMax() throws Exception { Configuration conf = new Configuration(); JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); MRAppMetrics mrAppMetrics = MRAppMetrics.create(); JobImpl job = new JobImpl(jobId, ApplicationAttemptId.newInstance( ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class), null, new JobTokenSecretManager(), new Credentials(), null, null, mrAppMetrics, null, true, null, 0, null, null, null, null); InitTransition initTransition = new InitTransition() { @Override protected TaskSplitMetaInfo[] createSplits(JobImpl job, JobId jobId) { throw new YarnRuntimeException(EXCEPTIONMSG); } }; JobEvent mockJobEvent = mock(JobEvent.class); JobStateInternal jobSI = initTransition.transition(job, mockJobEvent); Assert.assertTrue("When init fails, return value from InitTransition.transition should equal NEW.", jobSI.equals(JobStateInternal.NEW)); Assert.assertTrue("Job diagnostics should contain YarnRuntimeException", job.getDiagnostics().toString().contains("YarnRuntimeException")); Assert.assertTrue("Job diagnostics should contain " + EXCEPTIONMSG, job.getDiagnostics().toString().contains(EXCEPTIONMSG)); }
private boolean testUberDecision(Configuration conf) { JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); MRAppMetrics mrAppMetrics = MRAppMetrics.create(); JobImpl job = new JobImpl(jobId, ApplicationAttemptId.newInstance( ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class), null, new JobTokenSecretManager(), new Credentials(), null, null, mrAppMetrics, null, true, null, 0, null, null, null, null); InitTransition initTransition = getInitTransition(2); JobEvent mockJobEvent = mock(JobEvent.class); initTransition.transition(job, mockJobEvent); boolean isUber = job.isUber(); return isUber; }
@Override protected void setup(JobImpl job) throws IOException { super.setup(job); job.conf.setInt(MRJobConfig.NUM_REDUCES, reduces); job.remoteJobConfFile = new Path("test"); } @Override