while (executor.getActiveCount() < numOfConcurrentJobs && (toRun = dao.removeFromQueue()) != null) { CallableJob cj = assembleCallableJob(toRun, START_ATTEMPT_NUM); submitJob(cj); handleReruns( new ArrayList<>(getFailedQueries(LIMIT_JOB_RUNS).values()), AgentConsumer.maxReruns, this.waitBeforeRetrySeconds);
public void init() { LOG.info("Cleaning previously running jobs..."); cleanupPreviouslyRunningJobs(dao, dao.getRunningJobs()); LOG.info("Finished cleaning previously running jobs..."); me.start(); }
protected void handleException(Exception ex) { String jobName = plannedJob.getJobSpec().getName(); LOG.error(ex); setExceptionMessage(ex.getMessage()); JobSpec spec = plannedJob.getJobSpec(); if (spec.getStatusEmail() != null && spec.getStatusEmail().size() > 0 && !spec.getStatusEmail().get(0).equals("")) { for (String statusEmail : spec.getStatusEmail()) { MailInfo errMailInfo = new MailInfo(mailInfo.from, mailInfo.fromName, statusEmail, String.format("%s creator", jobName)); AgentConsumer.sendErrorReport(spec, replacedCode, ex, jobId, hostname, errMailInfo, session, attemptNumber); } } else { AgentConsumer.sendErrorReport(spec, replacedCode, ex, jobId, hostname, mailInfo, session, attemptNumber); } reporting.mark("chronos.query." + jobName + "." + "failed"); setStatus(1); }
@Override public void run() { try { LOG.info( String.format("Sleeping for %d seconds before retrying %s", waitBeforeRerun, jobName)); Thread.sleep(1000L * waitBeforeRerun); } catch (InterruptedException e) { LOG.info("rerunning job was interrupted..."); } finally { final CallableJob toResubmit = assembleCallableJob(pj, attempt + 1); submitJob(toResubmit); synchronized (pendingReruns) { pendingReruns.remove(pj); } } } };
consumer.submitJob(cj); Long id = cj.getJobId(); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 5, jobRuns.values().size()); assertEquals(5, consumer.getFinishedJobs(limit).size()); Map<Long, CallableJob> success = consumer.getSuccesfulQueries(limit); assertEquals("successful Queries:" + success, 0, success.size()); assertEquals(false, consumer.getFinishedJobs(limit).get(nextId).isSuccess());
@Test public void testCleanupPreviouslyRunningJobs() { JobSpec aJob = TestAgent.getTestJob("Mary Wollstonecraft", dao); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); SleepyCallableQuery cj = new SleepyCallableQuery( pj, dao, reporting, "example.com", mailInfo, null, null, drivers.get(0), 1, 10000); consumer.submitJob(cj); doSleep(); assertEquals(0, consumer.getFailedQueries(limit).size()); assertEquals(1, dao.getRunningJobs().size()); AgentConsumer.cleanupPreviouslyRunningJobs(dao, dao.getRunningJobs()); assertEquals(0, dao.getRunningJobs().size()); assertEquals(1, consumer.getFailedQueries(limit).size()); }
@Test(timeout=10000) public void testBasicLarge() throws Exception { int count = 100; for (int i = 1; i <= count; i++) { JobSpec aJob = getTestJob(String.valueOf(i), dao); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } } runRunnable(agentDriver); assertEquals(count, dao.getQueue(null).size()); waitUntilJobsFinished(consumer, count); assertEquals(0, dao.getQueue(null).size()); assertEquals(count, dao.getJobRuns(null, count).size()); assertEquals(0, consumer.getFailedQueries(count).size()); assertEquals(count, consumer.getFinishedJobs(count).size()); assertEquals(count, consumer.getSuccesfulQueries(count).size()); }
@Test public void testGetFailedQueries() { JobSpec aJob = TestAgent.getTestJob("David Foster Wallace", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); assertEquals(true, cj.isDone()); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); assertEquals(1, consumer.getFinishedJobs(limit).size()); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 0, 1); consumer.writeReportToLocal(folder.getRoot().getPath()); consumer.SLEEP_FOR = 20; AgentConsumer.setShouldSendErrorReports(false); }
@Ignore public void testSendErrorReport() { MailInfo testInfo = new MailInfo("", "", "my.test.email@huffingtonpost.com", ""); JobSpec aJob = new JobSpec(); aJob.setName("A Unit Test Job"); aJob.setCode("SELECT * FROM NOT_A_TABLE" + "\r\nWHERE DT >= 0;"); Exception ex = new Exception("SQL error, blah blah blah..."); AgentConsumer.setShouldSendErrorReports(true); for (int i = 1; i < 6; i++) { AgentConsumer.sendErrorReport(aJob, aJob.getCode(), ex, 4L, "testing.hostname.com", testInfo, Session.getDefaultInstance(null, null), i); } assertTrue(true); }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
consumer.submitJob(cj); totalJobs++; Map<Long, CallableJob> finished = consumer.getFinishedJobs(limit); TestAgent.runRunnable(consumer); assertEquals(numOfConcurrentJobs, dao.getRunningJobs().size()); assertEquals(0, consumer.getFinishedJobs(limit).size()); assertEquals(totalJobs, consumer.getFinishedJobs(limit).size());
"example.com", mailInfo, null, drivers.get(0), folder.getRoot().getPath(), 1); consumer.submitJob(cj); TestAgent.runRunnable(consumer); CallableQuery.getJobReportDir(consumer.getReportRootPath(), pj); String reportPath = CallableQuery.getJobReportPath(reportJobPath, pj);
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 1, 1); consumer.SLEEP_FOR = 1; AgentConsumer.setShouldSendErrorReports(false); }
public static void waitForFail(AgentConsumer c, int count) { while (c.getFailedQueries(AgentConsumer.LIMIT_JOB_RUNS).size() != count) { try { Thread.sleep(100); } catch (Exception ex) { ex.printStackTrace();} } }
@After public void cleanup() throws Exception { consumer.close(); }
@Override public void run() { while (isAlive) { try { doRun(); } catch (RuntimeException ex) { LOG.error("Exception when running consumer:", ex); } } }
public static void waitUntilJobsFinished(AgentConsumer c, int count) { while (c.getFinishedJobs(AgentConsumer.LIMIT_JOB_RUNS).size() != count) { runRunnable(c); try { Thread.sleep(100); } catch (Exception ignore) { } } }
continue; CallableJob latest = getLatestMatching(pj, LIMIT_JOB_RUNS); if (latest != null) { final int attempt = latest.getAttemptNumber();
@Test public void testJobNoResubmit() throws BackendException { consumer.SLEEP_FOR = 1; JobSpec aJob = TestAgent.getTestJob("Hannah Arendt", dao); aJob.setCode("not a valid query..."); aJob.setShouldRerun(false); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); // let the job run and fail TestAgent.waitUntilJobsFinished(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); // See if the job retries TestAgent.runRunnable(consumer); TestAgentConsumer.doSleep(); assertEquals(0, dao.getRunningJobs().size()); Map<Long, CallableJob> jobRuns = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 1, jobRuns.values().size()); assertEquals(1, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); }