@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 1, 1); consumer.SLEEP_FOR = 1; AgentConsumer.setShouldSendErrorReports(false); }
@Test public void testCancelPendingJob() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Foucault", dao); aJob.setShouldRerun(true); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); dao.addToQueue(pj); int actual = dao.cancelJob(pj); assertEquals(1, actual); actual = dao.cancelJob(pj); assertEquals(0, actual); TestAgent.runRunnable(consumer); assertEquals(0, dao.getRunningJobs().size()); assertEquals(0, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFailedQueries(limit).size()); }
public void init() throws BackendException { this.setDataSource(H2TestUtil.getDataSource()); try { conn = newConnection(); cleanupDb(); setUpDb(); } catch (ClassNotFoundException | SQLException ex) { throw new RuntimeException(ex); } super.init(); }
@Test(timeout=10000) public void testBasicLarge() throws Exception { int count = 100; for (int i = 1; i <= count; i++) { JobSpec aJob = getTestJob(String.valueOf(i), dao); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } } runRunnable(agentDriver); assertEquals(count, dao.getQueue(null).size()); waitUntilJobsFinished(consumer, count); assertEquals(0, dao.getQueue(null).size()); assertEquals(count, dao.getJobRuns(null, count).size()); assertEquals(0, consumer.getFailedQueries(count).size()); assertEquals(count, consumer.getFinishedJobs(count).size()); assertEquals(count, consumer.getSuccesfulQueries(count).size()); }
dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); assertEquals(numOfConcurrentJobs, dao.getRunningJobs().size()); JobSpec aJob = TestAgent.getTestJob("Joblah " + String.valueOf(numOfConcurrentJobs+i), dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(aJob.getId()), Utils.getCurrentTime()); dao.addToQueue(pj); totalJobs++; assertEquals(numOfConcurrentJobs, dao.getQueue(null).size()); TestAgent.runRunnable(consumer); assertEquals(numOfConcurrentJobs, dao.getRunningJobs().size()); assertEquals(0, consumer.getFinishedJobs(limit).size()); assertEquals(0, dao.getRunningJobs().size()); assertEquals(totalJobs, consumer.getFinishedJobs(limit).size());
"CREATE TABLE %s AS SELECT time, url, type FROM %s;", aJob.getResultTable(), H2TestJobDaoImpl.testTableName)); dao.createJob(aJob); runRunnable(agentDriver); runRunnable(consumer); dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); String name = value.getPlannedJob().getJobSpec().getName(); CallableJob cj = consumer.assembleCallableJob(value.getPlannedJob(), 1); dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(2L); Assert.assertEquals(name, rerun.getPlannedJob().getJobSpec().getName()); actualResults = dao.getJobResults(aJob, 100); } catch (Exception ex) { ex.printStackTrace(); } assertEquals(expectedResults, actualResults); JobSpec actualJob = dao.getJob(aJob.getId()); assertEquals(aJob, actualJob);
@Test public void testJobResubmitSuccessUpdateQuery() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Robert Frank", dao); String tableName = "table_dne_yet"; aJob.setCode(String.format("select * from %s", tableName)); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); Long nextId = cj.getJobId()+1; aJob = dao.getJob(aJob.getId()); aJob.setCode("show tables;"); dao.updateJob(aJob); TestAgent.runRunnable(consumer); doSleep(); assertEquals(0, dao.getRunningJobs().size()); assertEquals(2, consumer.getFinishedJobs(limit).size()); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFinishedJobs(limit).get(nextId).getStatus().get()); } @Test
JobSpec aJob = TestAgent.getTestJob("Simone de Beauvoir", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); executor.shutdownNow(); assertEquals(0, dao.getRunningJobs().size()); Map<Long, CallableJob> jobRuns = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 5, jobRuns.values().size()); assertEquals(5, consumer.getFinishedJobs(limit).size());
@Test public void testCleanupPreviouslyRunningJobs() { JobSpec aJob = TestAgent.getTestJob("Mary Wollstonecraft", dao); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); SleepyCallableQuery cj = new SleepyCallableQuery( pj, dao, reporting, "example.com", mailInfo, null, null, drivers.get(0), 1, 10000); consumer.submitJob(cj); doSleep(); assertEquals(0, consumer.getFailedQueries(limit).size()); assertEquals(1, dao.getRunningJobs().size()); AgentConsumer.cleanupPreviouslyRunningJobs(dao, dao.getRunningJobs()); assertEquals(0, dao.getRunningJobs().size()); assertEquals(1, consumer.getFailedQueries(limit).size()); }
@Test(timeout=2000) public void testBasic() throws Exception { String jobName = "Jean Paul Sartre"; for (JobSpec aJob : new JobSpec[]{ getTestJob(jobName, dao), getTestScript(jobName, dao) }) { aJob.setResultTable(null); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); dao.deleteJob(aJob.getId()); } }
@Test public void testScriptRun() { JobSpec aJob = getTestScript("Chenoweth", dao); aJob.setCode("echo 'hi'; echo 'bye';"); dao.createJob(aJob); runRunnable(agentDriver); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals("", actual.getExceptionMessage().get()); assertEquals(true, actual.isSuccess()); }
@Test(timeout=2000) public void testBasicWithFail() throws Exception { String resultTable = "SHOULDNT_EXIST"; // make sure table doesn't exist dropTable(resultTable); JobSpec aJob = getTestJob("Michael Scott", dao); aJob.setResultTable(resultTable); // Second query should not be executed since the first one fails aJob.setCode("select * from something_that_fails; " + String.format("CREATE TABLE %s AS SELECT time, url, type FROM %s;", aJob.getResultTable(), H2TestJobDaoImpl.testTableName)); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); List<String> actual = getResults(aJob); assertEquals(new ArrayList<String>(), actual); assertEquals(false, dao.showTables().contains(resultTable)); }
String tableName = "table_dne_yet"; aJob.setCode(String.format("select * from %s", tableName)); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); dao.execute( String.format("CREATE TABLE %s (blah TEXT)", tableName)); assertEquals(0, dao.getRunningJobs().size()); assertEquals(2, consumer.getFinishedJobs(limit).size()); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFinishedJobs(limit).get(nextId).getStatus().get()); dao.execute( String.format("DROP TABLE IF EXISTS %s", tableName));
@Test public void testBasicDependent() throws Exception { String jobName = "TE Lawrence"; JobSpec parent = getTestJob(jobName, dao); JobSpec child = getTestJob(jobName + " arabia", dao); child.setCronString(null); try { dao.createJob(parent); child.setParent(parent.getId()); long id = dao.createJob(child); } catch (Exception ex) { ex.printStackTrace(); } TestAgent.runRunnable(agentDriver); assertEquals(1, dao.getQueue(null).size()); runRunnable(consumer); Thread.sleep(100); // child job should have been queued assertEquals(1, dao.getQueue(null).size()); runRunnable(consumer); waitUntilJobsFinished(consumer, 2); assertEquals(2, consumer.getSuccesfulQueries(AgentConsumer.LIMIT_JOB_RUNS) .values().size()); } }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
aJob.setResultQuery("select * from test111 limit 10"); aJob.setResultEmail(null); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); id1, CallableQuery.TAB, aval1, CallableQuery.TAB)}; assertArrayEquals(expected, getLines(reportPath)); dao.execute("DROP TABLE IF EXISTS test111");
@Before public void setUp() throws Exception { dao = new H2TestJobDaoImpl(); dao.init(); }
private void dropTable(String aTable) { try { dao.execute(String.format("DROP TABLE IF EXISTS %s", aTable)); } catch (Exception ex) { ex.printStackTrace(); } }
@Override public void close() { cleanupDb(); super.close(); try { conn.close(); } catch (SQLException ex) { throw new RuntimeException(ex); } } }
private boolean areAllFuturesDone(AgentConsumer consumer){ boolean result = true; for (Entry<Long, CallableJob> i : dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).entrySet()){ if (!i.getValue().isDone()){ result = false; } } return result; }