protected void end() { String jobName = plannedJob.getJobSpec().getName(); finish.set(System.currentTimeMillis()); reporting.histogram("chronos.query." + jobName + "." + "querytime", finish.get() - start.get()); dao.updateJobRun(this); final Long jobId = plannedJob.getJobSpec().getId(); List<JobSpec> children = dao.getChildren(jobId); for (JobSpec aChild : children) { dao.addToQueue(new PlannedJob(aChild, plannedJob.getReplaceTime())); } }
@Override public void close() throws IOException { super.close(); if (dao != null) { dao.close(); } }
public void submitJob(final CallableJob cj) { dao.createJobRun(cj); final Future<Void> future = executor.submit(cj); reporter.mark("chronos.agentconsumer.submitted"); }
public void doRun() { DateTime now = Utils.getCurrentTime(); List<JobSpec> jobs = dao.getJobs(); for (JobSpec aJob : jobs) { if (shouldJobRun(aJob, now)) { LOG.info("Adding job to queue:" + aJob); dao.addToQueue(new PlannedJob(aJob, now)); } } try { LOG.info(String.format("Sleeping for %d seconds...", (SLEEP_FOR / 1000))); Thread.sleep(SLEEP_FOR); } catch (InterruptedException e) { e.printStackTrace(); } }
public void doRun() { List<PlannedJob> queue = dao.getQueue(null); boolean zeroInQueue = queue.size() == 0; int running = dao.getRunningJobs().size(); boolean maxJobsRunning = running >= numOfConcurrentJobs; if (zeroInQueue || maxJobsRunning) { PlannedJob toRun; while (executor.getActiveCount() < numOfConcurrentJobs && (toRun = dao.removeFromQueue()) != null) { CallableJob cj = assembleCallableJob(toRun, START_ATTEMPT_NUM); submitJob(cj);
@Test public void testQueueLarge() { int count = 1000; for (int i = 1; i <= count; i++) { JobSpec aJob = TestAgent.getTestJob("Vincent Van Gogh", dao); long id = dao.createJob(aJob); aJob = dao.getJob(id); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); dao.addToQueue(pj); } assertEquals(count, dao.getQueue(null).size()); }
@Test public void testGetJobRuns() { List<Long> ids = new ArrayList<>(); Map<Long, CallableJob> expected = new HashMap<>(); for (String name : new String[] { "Wifredo Lam", "Rene Magritte" }) { JobSpec aJob = TestAgent.getTestJob(name, dao); try { dao.createJob(aJob); aJob = dao.getJob(aJob.getId()); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); dao.createJobRun(cj); ids.add(aJob.getId()); expected.put(aJob.getId(), cj); } catch (Exception ex) { ex.printStackTrace(); } } for (Long id : ids) { Map<Long, CallableJob> got = dao.getJobRuns(id, limit); assertEquals(1, got.size()); assertEquals(expected.get(id), got.get(id)); } Map<Long, CallableJob> got = dao.getJobRuns(null, limit); assertEquals(expected, got); }
public void run() { for (int i = 0; i < size; i++) { JobSpec job = TestAgent.getTestJob(UUID.randomUUID().toString(), dao); long id = dao.createJob(job); job = dao.getJob(id); PlannedJob aJob = new PlannedJob(job, Utils.getCurrentTime()); try { dao.addToQueue(aJob); } catch (Exception ex) { ex.printStackTrace(); } expected.add(aJob); } } };
@Test public void deleteJob() { JobSpec expected = TestAgent.getTestJob("Jane Austen", dao); try { dao.createJob(expected); expected.setName("Virginia Woolf"); Thread.sleep(10); dao.updateJob(expected); dao.deleteJob(expected.getId()); } catch (Exception ex) { ex.printStackTrace(); } JobSpec actual = dao.getJob(expected.getId()); assertEquals(null, actual); }
@Test public void testGetJobVersions() { JobSpec expected = TestAgent.getTestJob("Franz Kafka", dao); try { dao.createJob(expected); expected = dao.getJob(expected.getId()); expected.setDescription("Metamorphosis"); dao.updateJob(expected); } catch (Exception ex) { ex.printStackTrace(); } JobSpec actual = dao.getJob(expected.getId()); assertEquals(expected, actual); List<JobSpec> versions = dao.getJobVersions(actual.getId()); assertEquals(2, versions.size()); }
@Test public void updateJob() { JobSpec expected = TestAgent.getTestJob("Thomas Bernhard", dao); try { dao.createJob(expected); expected.setDescription("The Woodcutters"); Thread.sleep(10); dao.updateJob(expected); } catch (Exception ex) { ex.printStackTrace(); } JobSpec actual = dao.getJob(expected.getId()); assertEquals(expected, actual); }
synchronized public Map<Long, CallableJob> getFailedQueries(int limit) { Map<Long, CallableJob> toRet = new TreeMap<>(); Map<Long, CallableJob> runs = dao.getJobRuns(null, limit); for (Entry<Long, CallableJob> entry : runs.entrySet()) { Long key = entry.getKey(); CallableJob value = entry.getValue(); if (isJobFailed(value)) { toRet.put(key, value); } } return toRet; }
@Test public void testGetJobs() { List<JobSpec> expected = new ArrayList<>(); for (String aName : new String[]{ "Allen Ginsberg", "Louis Gluck"}) { JobSpec aJob = TestAgent.getTestJob(aName, dao); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } expected.add(aJob); } List<JobSpec> actual = dao.getJobs(); assertEquals(expected, actual); }
public static void cleanupPreviouslyRunningJobs( JobDao dao, Map<Long, CallableJob> jobRuns) { for (Entry<Long, CallableJob> job : jobRuns.entrySet()) { CallableJob value = job.getValue(); value.getFinish().set(System.currentTimeMillis()); dao.updateJobRun(value); } }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
public void setReplacedCode() { Long id = plannedJob.getJobSpec().getId(); JobSpec currJob = dao.getJob(id); this.replacedCode = QueryReplaceUtil.replaceDateValues( currJob.getCode(), plannedJob.getReplaceTime()); }
List<PlannedJob> actual = dao.getQueue(null); Collections.sort(actual); assertEquals(size*2, actual.size());
JobSpec childChildJob = TestAgent.getTestJob("D", dao); try { dao.createJob(parent); childJob.setParent(parent.getId()); dao.createJob(childJob); childJob2.setParent(parent.getId()); dao.createJob(childJob2); childChildJob.setParent(childJob.getId()); dao.createJob(childChildJob); } catch (Exception ex) { ex.printStackTrace(); } eChild.getChildren().add(eChildChild); JobNode actual = dao.getTree(parent.getId(), null); assertEquals(expected, actual); actual = dao.getTree(childJob.getId(), null); assertEquals(expected, actual);
JobSpec aJob = TestAgent.getTestJob("Broken Social Scene", dao); aJob.setResultQuery("select * from blah where dt=${YYYMMDDHH}"); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cq = new CallableQuery(pj, dao,