private static boolean existsJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement queryStatement = connection .prepareStatement(JOB_EXECUTION_QUERY_BY_JOB_ID_STATEMENT_TEMPLATE)) { queryStatement.setString(1, info.getJobId()); try (ResultSet resultSet = queryStatement.executeQuery()) { return resultSet.next(); } } }
private Map<String, JobExecutionInfo> getJobExecutionInfos(Connection connection, List<String> jobIds) throws SQLException { Map<String, JobExecutionInfo> jobExecutionInfos = Maps.newLinkedHashMap(); if (jobIds != null && jobIds.size() > 0) { String template = String.format(JOB_EXECUTION_QUERY_BY_JOB_ID_STATEMENT_TEMPLATE, getInPredicate(jobIds.size())); int index = 1; try (PreparedStatement jobExecutionQueryStatement = connection.prepareStatement(template)) { for (String jobId : jobIds) { jobExecutionQueryStatement.setString(index++, jobId); } try (ResultSet jobRs = jobExecutionQueryStatement.executeQuery()) { while (jobRs.next()) { JobExecutionInfo jobExecutionInfo = resultSetToJobExecutionInfo(jobRs); jobExecutionInfos.put(jobExecutionInfo.getJobId(), jobExecutionInfo); } } } } return jobExecutionInfos; }
private void upsertJobProperties(Connection connection, JobExecutionInfo jobExecutionInfo) throws SQLException { if (jobExecutionInfo.hasJobProperties()) { Optional<PreparedStatement> upsertStatement = Optional.absent(); int batchSize = 0; for (Map.Entry<String, String> property : jobExecutionInfo.getJobProperties().entrySet()) { if (!upsertStatement.isPresent()) { upsertStatement = Optional.of(connection.prepareStatement(JOB_PROPERTY_UPSERT_STATEMENT_TEMPLATE)); } addPropertyToBatch(upsertStatement.get(), property.getKey(), property.getValue(), jobExecutionInfo.getJobId()); if (batchSize++ > 1000) { executeBatches(upsertStatement); upsertStatement = Optional.absent(); batchSize = 0; } } executeBatches(upsertStatement); } }
private void upsertJobMetrics(Connection connection, JobExecutionInfo jobExecutionInfo) throws SQLException { if (jobExecutionInfo.hasMetrics()) { Optional<PreparedStatement> upsertStatement = Optional.absent(); int batchSize = 0; for (Metric metric : jobExecutionInfo.getMetrics()) { if (!upsertStatement.isPresent()) { upsertStatement = Optional.of(connection.prepareStatement(JOB_METRIC_UPSERT_STATEMENT_TEMPLATE)); } addMetricToBatch(upsertStatement.get(), metric, jobExecutionInfo.getJobId()); if (batchSize++ > 1000) { executeBatches(upsertStatement); upsertStatement = Optional.absent(); batchSize = 0; } } executeBatches(upsertStatement); } }
data.add(Arrays.asList("Job Id", jobExecutionInfo.getJobId())); data.add(Arrays.asList("State", jobExecutionInfo.getState().toString())); data.add(Arrays.asList("Completed/Launched Tasks",
for (JobExecutionInfo jobInfo : jobExecutionInfos) { List<String> entry = new ArrayList<>(); entry.add(jobInfo.getJobId()); entry.add(jobInfo.getState().toString()); entry.add(extractJobSchedule(jobInfo));
private void upsertJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobName()); Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement upsertStatement = connection.prepareStatement(JOB_EXECUTION_UPSERT_STATEMENT_TEMPLATE)) { int index = 0; upsertStatement.setString(++index, info.getJobName()); upsertStatement.setString(++index, info.getJobId()); upsertStatement.setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : null, getCalendarUTCInstance()); upsertStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : null, getCalendarUTCInstance()); upsertStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); upsertStatement.setString(++index, info.hasState() ? info.getState().name() : null); upsertStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() : -1); upsertStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks() : -1); upsertStatement.setString(++index, getLauncherType(info)); upsertStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() : null); upsertStatement.executeUpdate(); } }
private static void updateJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement updateStatement = connection.prepareStatement(JOB_EXECUTION_UPDATE_STATEMENT_TEMPLATE)) { int index = 0; updateStatement .setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); updateStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); updateStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); updateStatement.setString(++index, info.hasState() ? info.getState().name() : null); updateStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() : -1); updateStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks() : -1); updateStatement.setString(++index, info.hasLauncherType() ? info.getLauncherType().name() : null); updateStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() : null); updateStatement.setString(++index, info.getJobId()); updateStatement.executeUpdate(); } }
private static void insertJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobName()); Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement insertStatement = connection.prepareStatement(JOB_EXECUTION_INSERT_STATEMENT_TEMPLATE)) { int index = 0; insertStatement.setString(++index, info.getJobName()); insertStatement.setString(++index, info.getJobId()); insertStatement .setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); insertStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); insertStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); insertStatement.setString(++index, info.hasState() ? info.getState().name() : null); insertStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() : -1); insertStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks() : -1); insertStatement.setString(++index, info.hasLauncherType() ? info.getLauncherType().name() : null); insertStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() : null); insertStatement.executeUpdate(); } }
@Test(dependsOnMethods = {"testUpdate"}) public void testQueryByJobId() throws IOException { JobExecutionQuery queryByJobId = new JobExecutionQuery(); queryByJobId.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId.setId(JobExecutionQuery.Id.create(this.expectedJobExecutionInfos.get(0).getJobId())); List<JobExecutionInfo> result = this.jobHistoryStore.get(queryByJobId); Assert.assertEquals(result.size(), 1); JobExecutionInfo actual = result.get(0); JobExecutionInfo expected = this.expectedJobExecutionInfos.get(0); assertJobExecution(actual, expected); }
for (Metric metric : jobExecutionInfo.getMetrics()) { boolean insert = !existsMetric(connection, JOB_METRIC_EXIST_QUERY_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), metric); updateMetric(connection, insert ? JOB_METRIC_INSERT_STATEMENT_TEMPLATE : JOB_METRIC_UPDATE_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), metric, insert); for (Map.Entry<String, String> entry : jobExecutionInfo.getJobProperties().entrySet()) { boolean insert = !existsProperty(connection, JOB_PROPERTY_EXIST_QUERY_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), entry.getKey()); updateProperty(connection, insert ? JOB_PROPERTY_INSERT_STATEMENT_TEMPLATE : JOB_PROPERTY_UPDATE_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), entry.getKey(), entry.getValue(), insert);
@Test(dependsOnMethods = {"testSetAndGet"}) public void testToJobExecutionInfo() { JobExecutionInfo jobExecutionInfo = this.jobState.toJobExecutionInfo(); Assert.assertEquals(jobExecutionInfo.getJobName(), "TestJob"); Assert.assertEquals(jobExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(jobExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(jobExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(jobExecutionInfo.getDuration().longValue(), 1000L); Assert.assertEquals(jobExecutionInfo.getState().name(), JobState.RunningState.COMMITTED.name()); Assert.assertEquals(jobExecutionInfo.getLaunchedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getCompletedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getJobProperties().get("foo"), "bar"); List<String> taskStateIds = Lists.newArrayList(); for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) { Assert.assertEquals(taskExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000); Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name()); Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar"); taskStateIds.add(taskExecutionInfo.getTaskId()); } Collections.sort(taskStateIds); Assert.assertEquals(taskStateIds, Lists.newArrayList("TestTask-0", "TestTask-1", "TestTask-2")); } }
@Test public void testBadGet() throws Exception { JobExecutionQuery queryByJobId = new JobExecutionQuery(); queryByJobId.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId.setId(JobExecutionQuery.Id.create(this.expected1.getJobId() + "1")); JobExecutionQueryResult result = this.client.get(queryByJobId); Assert.assertTrue(result.getJobExecutions().isEmpty()); }
Assert.assertEquals(actual.getJobId(), this.expectedJobExecutionInfos.get(0).getJobId()); Assert.assertEquals(actual.getTaskExecutions().size(), 1); Assert.assertEquals(actual.getTaskExecutions().get(0).getTable(), Assert.assertEquals(actual.getJobId(), this.expectedJobExecutionInfos.get(1).getJobId()); Assert.assertEquals(actual.getTaskExecutions().size(), 1); Assert.assertEquals(actual.getTaskExecutions().get(0).getTable(), Assert.assertEquals(actual.getJobId(), this.expectedJobExecutionInfos.get(1).getJobId()); Assert.assertEquals(result.size(), 1); Assert.assertEquals(actual.getTaskExecutions().size(), 1);
@Test public void testGet() throws Exception { JobExecutionQuery queryByJobId = new JobExecutionQuery(); queryByJobId.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId.setId(JobExecutionQuery.Id.create(this.expected1.getJobId())); JobExecutionQueryResult result = this.client.get(queryByJobId); JobExecutionInfoArray jobExecutionInfos = result.getJobExecutions(); Assert.assertEquals(jobExecutionInfos.size(), 1); JobExecutionInfo actual = jobExecutionInfos.get(0); assertJobExecution(actual, this.expected1); }
@Test public void testBatchGet() throws Exception { JobExecutionQuery queryByJobId1 = new JobExecutionQuery(); queryByJobId1.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId1.setId(JobExecutionQuery.Id.create(this.expected1.getJobId())); JobExecutionQuery queryByJobId2 = new JobExecutionQuery(); queryByJobId2.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId2.setId(JobExecutionQuery.Id.create(this.expected2.getJobId())); List<JobExecutionQuery> queries = Lists.newArrayList(queryByJobId1, queryByJobId2); List<JobExecutionQueryResult> result = Lists.newArrayList(this.client.batchGet(queries)); Assert.assertEquals(result.size(), 2); Assert.assertEquals(result.get(0).getJobExecutions().size(), 1); Assert.assertEquals(result.get(1).getJobExecutions().size(), 1); JobExecutionInfo actual1 = result.get(0).getJobExecutions().get(0); JobExecutionInfo actual2 = result.get(1).getJobExecutions().get(0); if (actual1.getJobName().equals(this.expected1.getJobName())) { assertJobExecution(actual1, this.expected1); assertJobExecution(actual2, this.expected2); } else { assertJobExecution(actual1, this.expected2); assertJobExecution(actual2, this.expected1); } }
private static void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), -1L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
private void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), 0L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getLauncherType(), expected.getLauncherType()); Assert.assertEquals(actual.getTrackingUrl(), expected.getTrackingUrl()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0"); taskExecutionInfo1.setStartTime(System.currentTimeMillis()); taskExecutionInfo1.setState(TaskStateEnum.PENDING); taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1"); taskExecutionInfo2.setStartTime(System.currentTimeMillis()); taskExecutionInfo2.setState(TaskStateEnum.PENDING);
taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0"); taskExecutionInfo1.setStartTime(System.currentTimeMillis()); taskExecutionInfo1.setState(TaskStateEnum.PENDING); taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1"); taskExecutionInfo2.setStartTime(System.currentTimeMillis()); taskExecutionInfo2.setState(TaskStateEnum.PENDING);