private static void updateJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement updateStatement = connection.prepareStatement(JOB_EXECUTION_UPDATE_STATEMENT_TEMPLATE)) { int index = 0; updateStatement .setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); updateStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); updateStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); updateStatement.setString(++index, info.hasState() ? info.getState().name() : null); updateStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() : -1); updateStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks() : -1); updateStatement.setString(++index, info.hasLauncherType() ? info.getLauncherType().name() : null); updateStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() : null); updateStatement.setString(++index, info.getJobId()); updateStatement.executeUpdate(); } }
private static void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), -1L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
private JobExecutionInfo resultSetToJobExecutionInfo(ResultSet rs) throws SQLException { JobExecutionInfo jobExecutionInfo = new JobExecutionInfo(); jobExecutionInfo.setJobName(rs.getString("job_name")); jobExecutionInfo.setJobId(rs.getString("job_id")); try { jobExecutionInfo.setStartTime(rs.getTimestamp("start_time").getTime()); } catch (SQLException se) { jobExecutionInfo.setStartTime(0); jobExecutionInfo.setEndTime(rs.getTimestamp("end_time").getTime()); } catch (SQLException se) { jobExecutionInfo.setEndTime(0); jobExecutionInfo.setDuration(rs.getLong("duration")); String state = rs.getString("state"); if (!Strings.isNullOrEmpty(state)) { jobExecutionInfo.setState(JobStateEnum.valueOf(state)); jobExecutionInfo.setLaunchedTasks(rs.getInt("launched_tasks")); jobExecutionInfo.setCompletedTasks(rs.getInt("completed_tasks")); String launcherType = rs.getString("launcher_type"); if (!Strings.isNullOrEmpty(launcherType)) { jobExecutionInfo.setLauncherType(LauncherTypeEnum.valueOf(launcherType)); jobExecutionInfo.setTrackingUrl(trackingUrl);
private static boolean existsJobExecutionInfo(Connection connection, JobExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement queryStatement = connection .prepareStatement(JOB_EXECUTION_QUERY_BY_JOB_ID_STATEMENT_TEMPLATE)) { queryStatement.setString(1, info.getJobId()); try (ResultSet resultSet = queryStatement.executeQuery()) { return resultSet.next(); } } }
private JobExecutionInfo create(int index, boolean differentTableType) { JobExecutionInfo jobExecutionInfo = new JobExecutionInfo(); jobExecutionInfo.setJobName("TestJob" + index); jobExecutionInfo.setJobId(jobExecutionInfo.getJobName() + "_" + System.currentTimeMillis()); jobExecutionInfo.setStartTime(System.currentTimeMillis()); jobExecutionInfo.setState(JobStateEnum.PENDING); jobExecutionInfo.setLaunchedTasks(2); jobExecutionInfo.setCompletedTasks(0); jobExecutionInfo.setLauncherType(LauncherTypeEnum.LOCAL); jobExecutionInfo.setTrackingUrl("localhost"); jobMetric1.setValue("100"); jobMetrics.add(jobMetric1); jobExecutionInfo.setMetrics(jobMetrics); jobExecutionInfo.setJobProperties(new StringMap(jobProperties)); taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0"); taskExecutionInfo1.setStartTime(System.currentTimeMillis()); taskExecutionInfo1.setState(TaskStateEnum.PENDING); taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1"); taskExecutionInfo2.setStartTime(System.currentTimeMillis()); taskExecutionInfo2.setState(TaskStateEnum.PENDING); jobExecutionInfo.setTaskExecutions(taskExecutionInfos); this.expectedJobExecutionInfos.add(jobExecutionInfo);
JobExecutionInfo jobExecutionInfo = new JobExecutionInfo(); jobExecutionInfo.setJobName(this.jobName); jobExecutionInfo.setJobId(this.jobId); if (this.startTime > 0) { jobExecutionInfo.setStartTime(this.startTime); jobExecutionInfo.setEndTime(this.endTime); jobExecutionInfo.setDuration(this.duration); jobExecutionInfo.setState(JobStateEnum.valueOf(this.state.name())); jobExecutionInfo.setLaunchedTasks(this.taskCount); jobExecutionInfo.setCompletedTasks(this.getCompletedTasks()); jobExecutionInfo.setLauncherType(getLauncherType()); if (getTrackingURL().isPresent()) { jobExecutionInfo.setTrackingUrl(getTrackingURL().get()); taskExecutionInfos.add(taskState.toTaskExecutionInfo()); jobExecutionInfo.setTaskExecutions(taskExecutionInfos); jobExecutionInfo.setMetrics(metricArray); jobExecutionInfo.setJobProperties(new StringMap(jobProperties));
List<String> flags = Arrays.asList("", "-"); data.add(Arrays.asList("Job Name", jobExecutionInfo.getJobName())); data.add(Arrays.asList("Job Id", jobExecutionInfo.getJobId())); data.add(Arrays.asList("State", jobExecutionInfo.getState().toString())); data.add(Arrays.asList("Completed/Launched Tasks", String.format("%d/%d", jobExecutionInfo.getCompletedTasks(), jobExecutionInfo.getLaunchedTasks()))); data.add(Arrays.asList("Start Time", dateTimeFormatter.print(jobExecutionInfo.getStartTime()))); data.add(Arrays.asList("End Time", dateTimeFormatter.print(jobExecutionInfo.getEndTime()))); data.add(Arrays.asList("Duration", jobExecutionInfo.getState() == JobStateEnum.COMMITTED ? periodFormatter .print(new Period(jobExecutionInfo.getDuration().longValue())) : "-")); data.add(Arrays.asList("Tracking URL", jobExecutionInfo.getTrackingUrl())); data.add(Arrays.asList("Launcher Type", jobExecutionInfo.getLauncherType().name())); .printTable(); JobInfoPrintUtils.printMetrics(jobExecutionInfo.getMetrics());
@Test(dependsOnMethods = {"testSetAndGet"}) public void testToJobExecutionInfo() { JobExecutionInfo jobExecutionInfo = this.jobState.toJobExecutionInfo(); Assert.assertEquals(jobExecutionInfo.getJobName(), "TestJob"); Assert.assertEquals(jobExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(jobExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(jobExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(jobExecutionInfo.getDuration().longValue(), 1000L); Assert.assertEquals(jobExecutionInfo.getState().name(), JobState.RunningState.COMMITTED.name()); Assert.assertEquals(jobExecutionInfo.getLaunchedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getCompletedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getJobProperties().get("foo"), "bar"); List<String> taskStateIds = Lists.newArrayList(); for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) { Assert.assertEquals(taskExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000); Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name()); Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar"); taskStateIds.add(taskExecutionInfo.getTaskId()); } Collections.sort(taskStateIds); Assert.assertEquals(taskStateIds, Lists.newArrayList("TestTask-0", "TestTask-1", "TestTask-2")); } }
for (JobExecutionInfo jobInfo : jobExecutionInfos) { List<String> entry = new ArrayList<>(); entry.add(jobInfo.getJobId()); entry.add(jobInfo.getState().toString()); entry.add(extractJobSchedule(jobInfo)); entry.add(jobInfo.getCompletedTasks().toString()); entry.add(jobInfo.getLaunchedTasks().toString()); entry.add(dateTimeFormatter.print(jobInfo.getStartTime())); entry.add(dateTimeFormatter.print(jobInfo.getEndTime())); entry.add(jobInfo.getState() == JobStateEnum.COMMITTED ? decimalFormatter.format(jobInfo.getDuration() / 1000.0) : "-"); data.add(entry);
@Test(dependsOnMethods = {"testInsert"}) public void testUpdate() throws IOException { for (JobExecutionInfo jobExecutionInfo : this.expectedJobExecutionInfos) { jobExecutionInfo.setEndTime(System.currentTimeMillis()); jobExecutionInfo.setDuration(jobExecutionInfo.getEndTime() - jobExecutionInfo.getStartTime()); jobExecutionInfo.setState(JobStateEnum.COMMITTED); jobExecutionInfo.setCompletedTasks(jobExecutionInfo.getLaunchedTasks()); for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) { taskExecutionInfo.setEndTime(jobExecutionInfo.getEndTime()); taskExecutionInfo.setDuration(taskExecutionInfo.getEndTime() - taskExecutionInfo.getStartTime()); taskExecutionInfo.setState(TaskStateEnum.COMMITTED); } this.jobHistoryStore.put(jobExecutionInfo); } }
if (jobExecutionInfo.hasMetrics()) { for (Metric metric : jobExecutionInfo.getMetrics()) { boolean insert = !existsMetric(connection, JOB_METRIC_EXIST_QUERY_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), metric); updateMetric(connection, insert ? JOB_METRIC_INSERT_STATEMENT_TEMPLATE : JOB_METRIC_UPDATE_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), metric, insert); if (jobExecutionInfo.hasJobProperties()) { for (Map.Entry<String, String> entry : jobExecutionInfo.getJobProperties().entrySet()) { boolean insert = !existsProperty(connection, JOB_PROPERTY_EXIST_QUERY_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), entry.getKey()); updateProperty(connection, insert ? JOB_PROPERTY_INSERT_STATEMENT_TEMPLATE : JOB_PROPERTY_UPDATE_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), entry.getKey(), entry.getValue(), insert); if (jobExecutionInfo.hasTaskExecutions()) { for (TaskExecutionInfo info : jobExecutionInfo.getTaskExecutions()) {
for (JobExecutionInfo jobInfo : jobExecutionInfos) { List<String> entry = new ArrayList<>(); entry.add(jobInfo.getJobName()); entry.add(jobInfo.getState().toString()); entry.add(dateTimeFormatter.print(jobInfo.getStartTime())); entry.add(dateTimeFormatter.print(jobInfo.getEndTime())); MetricArray metrics = jobInfo.getMetrics(); Double recordsProcessed = null; Double recordsFailed = null;
jobExecutionInfo.setMetrics(jobMetrics); jobPropertiesQueryStatement.setString(1, jobExecutionInfo.getJobId()); try (ResultSet jobPropertiesRs = jobPropertiesQueryStatement.executeQuery()) { Map<String, String> jobProperties = Maps.newHashMap(); jobExecutionInfo.setJobProperties(new StringMap(jobProperties)); jobExecutionInfo.setTaskExecutions(taskExecutionInfos);
private void upsertJobProperties(Connection connection, JobExecutionInfo jobExecutionInfo) throws SQLException { if (jobExecutionInfo.hasJobProperties()) { Optional<PreparedStatement> upsertStatement = Optional.absent(); int batchSize = 0; for (Map.Entry<String, String> property : jobExecutionInfo.getJobProperties().entrySet()) { if (!upsertStatement.isPresent()) { upsertStatement = Optional.of(connection.prepareStatement(JOB_PROPERTY_UPSERT_STATEMENT_TEMPLATE)); } addPropertyToBatch(upsertStatement.get(), property.getKey(), property.getValue(), jobExecutionInfo.getJobId()); if (batchSize++ > 1000) { executeBatches(upsertStatement); upsertStatement = Optional.absent(); batchSize = 0; } } executeBatches(upsertStatement); } }
if (jobExecutionInfo.hasTaskExecutions()) { upsertTaskExecutionInfos(connection, jobExecutionInfo.getTaskExecutions()); upsertTaskMetrics(connection, jobExecutionInfo.getTaskExecutions()); Optional<StringMap> jobProperties = Optional.absent(); if (jobExecutionInfo.hasJobProperties()) { jobProperties = Optional.of(jobExecutionInfo.getJobProperties()); upsertTaskProperties(connection, jobProperties, jobExecutionInfo.getTaskExecutions());
JobExecutionQuery.Id.create(this.expectedJobExecutionInfos.get(0).getTaskExecutions().get(0).getTable())); Assert.assertEquals(actual.getJobName(), this.expectedJobExecutionInfos.get(0).getJobName()); Assert.assertEquals(actual.getJobId(), this.expectedJobExecutionInfos.get(0).getJobId()); Assert.assertEquals(actual.getTaskExecutions().size(), 1); Assert.assertEquals(actual.getTaskExecutions().get(0).getTable(), this.expectedJobExecutionInfos.get(0).getTaskExecutions().get(0).getTable()); Assert.assertEquals(actual.getJobName(), this.expectedJobExecutionInfos.get(1).getJobName()); Assert.assertEquals(actual.getJobId(), this.expectedJobExecutionInfos.get(1).getJobId()); Assert.assertEquals(actual.getTaskExecutions().size(), 1); Assert.assertEquals(actual.getTaskExecutions().get(0).getTable(), this.expectedJobExecutionInfos.get(1).getTaskExecutions().get(0).getTable()); JobExecutionQuery.Id.create(this.expectedJobExecutionInfos.get(1).getTaskExecutions().get(1).getTable())); Assert.assertEquals(actual.getJobName(), this.expectedJobExecutionInfos.get(1).getJobName()); Assert.assertEquals(actual.getJobId(), this.expectedJobExecutionInfos.get(1).getJobId()); Assert.assertEquals(result.size(), 1); Assert.assertEquals(actual.getTaskExecutions().size(), 1); Assert.assertEquals(actual.getTaskExecutions().get(0).getTable(), this.expectedJobExecutionInfos.get(1).getTaskExecutions().get(1).getTable());
private void upsertJobMetrics(Connection connection, JobExecutionInfo jobExecutionInfo) throws SQLException { if (jobExecutionInfo.hasMetrics()) { Optional<PreparedStatement> upsertStatement = Optional.absent(); int batchSize = 0; for (Metric metric : jobExecutionInfo.getMetrics()) { if (!upsertStatement.isPresent()) { upsertStatement = Optional.of(connection.prepareStatement(JOB_METRIC_UPSERT_STATEMENT_TEMPLATE)); } addMetricToBatch(upsertStatement.get(), metric, jobExecutionInfo.getJobId()); if (batchSize++ > 1000) { executeBatches(upsertStatement); upsertStatement = Optional.absent(); batchSize = 0; } } executeBatches(upsertStatement); } }
String jobId = jobPropertiesRs.getString("job_id"); JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId); StringMap jobProperties = jobExecutionInfo.getJobProperties(GetMode.NULL); if (jobProperties == null) { jobProperties = new StringMap(Maps.<String, String>newHashMap()); jobExecutionInfo.setJobProperties(jobProperties);
private void addMetricsToJobExecutions(Connection connection, JobExecutionQuery query, Map<String, JobExecutionInfo> jobExecutionInfos) throws SQLException { if (query.isIncludeJobMetrics() && jobExecutionInfos.size() > 0) { String template = String.format(JOB_METRIC_QUERY_STATEMENT_TEMPLATE, getInPredicate(jobExecutionInfos.size())); int index = 1; try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(template)) { for (String jobId : jobExecutionInfos.keySet()) { jobMetricQueryStatement.setString(index++, jobId); } try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { while (jobMetricRs.next()) { String jobId = jobMetricRs.getString("job_id"); JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId); MetricArray metricArray = jobExecutionInfo.getMetrics(GetMode.NULL); if (metricArray == null) { metricArray = new MetricArray(); jobExecutionInfo.setMetrics(metricArray); } metricArray.add(resultSetToMetric(jobMetricRs)); } } } } }
@Test public void testBatchGet() throws Exception { JobExecutionQuery queryByJobId1 = new JobExecutionQuery(); queryByJobId1.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId1.setId(JobExecutionQuery.Id.create(this.expected1.getJobId())); JobExecutionQuery queryByJobId2 = new JobExecutionQuery(); queryByJobId2.setIdType(QueryIdTypeEnum.JOB_ID); queryByJobId2.setId(JobExecutionQuery.Id.create(this.expected2.getJobId())); List<JobExecutionQuery> queries = Lists.newArrayList(queryByJobId1, queryByJobId2); List<JobExecutionQueryResult> result = Lists.newArrayList(this.client.batchGet(queries)); Assert.assertEquals(result.size(), 2); Assert.assertEquals(result.get(0).getJobExecutions().size(), 1); Assert.assertEquals(result.get(1).getJobExecutions().size(), 1); JobExecutionInfo actual1 = result.get(0).getJobExecutions().get(0); JobExecutionInfo actual2 = result.get(1).getJobExecutions().get(0); if (actual1.getJobName().equals(this.expected1.getJobName())) { assertJobExecution(actual1, this.expected1); assertJobExecution(actual2, this.expected2); } else { assertJobExecution(actual1, this.expected2); assertJobExecution(actual2, this.expected1); } }