private static TaskExecutionInfo resultSetToTaskExecutionInfo(ResultSet rs) throws SQLException { TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo(); taskExecutionInfo.setTaskId(rs.getString("task_id")); taskExecutionInfo.setJobId(rs.getString("job_id")); try { taskExecutionInfo.setStartTime(rs.getTimestamp("start_time").getTime()); } catch (SQLException se) { taskExecutionInfo.setStartTime(0); taskExecutionInfo.setEndTime(rs.getTimestamp("end_time").getTime()); } catch (SQLException se) { taskExecutionInfo.setEndTime(0); taskExecutionInfo.setDuration(rs.getLong("duration")); String state = rs.getString("state"); if (!Strings.isNullOrEmpty(state)) { taskExecutionInfo.setState(TaskStateEnum.valueOf(state)); taskExecutionInfo.setFailureException(failureException); taskExecutionInfo.setLowWatermark(rs.getLong("low_watermark")); taskExecutionInfo.setHighWatermark(rs.getLong("high_watermark")); Table table = new Table(); String namespace = rs.getString("table_namespace"); table.setType(TableTypeEnum.valueOf(type)); taskExecutionInfo.setTable(table);
String taskId = taskMetricRs.getString("task_id"); TaskExecutionInfo taskExecutionInfo = taskExecutionInfos.get(jobId).get(taskId); MetricArray metricsArray = taskExecutionInfo.getMetrics(GetMode.NULL); if (metricsArray == null) { metricsArray = new MetricArray(); taskExecutionInfo.setMetrics(metricsArray);
String taskId = taskPropertiesRs.getString("task_id"); TaskExecutionInfo taskExecutionInfo = taskExecutionInfos.get(jobId).get(taskId); StringMap taskProperties = taskExecutionInfo.getTaskProperties(GetMode.NULL); if (taskProperties == null) { taskProperties = new StringMap(); taskExecutionInfo.setTaskProperties(taskProperties);
private void addTaskExecutionInfoToBatch(PreparedStatement upsertStatement, TaskExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasTaskId()); Preconditions.checkArgument(info.hasJobId()); int index = 0; upsertStatement.setString(++index, info.getTaskId()); upsertStatement.setString(++index, info.getJobId()); upsertStatement.setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : null, getCalendarUTCInstance()); upsertStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : null, getCalendarUTCInstance()); upsertStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); upsertStatement.setString(++index, info.hasState() ? info.getState().name() : null); upsertStatement.setString(++index, info.hasFailureException() ? info.getFailureException() : null); upsertStatement.setLong(++index, info.hasLowWatermark() ? info.getLowWatermark() : -1); upsertStatement.setLong(++index, info.hasHighWatermark() ? info.getHighWatermark() : -1); upsertStatement.setString(++index, info.hasTable() && info.getTable().hasNamespace() ? info.getTable().getNamespace() : null); upsertStatement.setString(++index, info.hasTable() && info.getTable().hasName() ? info.getTable().getName() : null); upsertStatement.setString(++index, info.hasTable() && info.getTable().hasType() ? info.getTable().getType().name() : null); upsertStatement.addBatch(); }
private static void assertTaskExecution(TaskExecutionInfo actual, TaskExecutionInfo expected) { Assert.assertEquals(actual.getJobId(), expected.getJobId()); Assert.assertEquals(actual.getTaskId(), expected.getTaskId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), -1L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLowWatermark(), expected.getLowWatermark()); Assert.assertEquals(actual.getHighWatermark(), expected.getHighWatermark()); Assert.assertEquals(actual.getTable(), expected.getTable()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getTaskProperties(), expected.getTaskProperties()); }
TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo(); taskExecutionInfo.setJobId(this.jobId); taskExecutionInfo.setTaskId(this.taskId); if (this.startTime > 0) { taskExecutionInfo.setStartTime(this.startTime); taskExecutionInfo.setEndTime(this.endTime); taskExecutionInfo.setDuration(this.duration); taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name())); if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)); taskExecutionInfo.setHighWatermark(this.getHighWaterMark()); table.setType(TableTypeEnum.valueOf(extract.getType().name())); taskExecutionInfo.setTable(table); taskExecutionInfo.setMetrics(metricArray); taskProperties.put(name, value); taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
TaskExecutionInfo taskExecutionInfo1 = new TaskExecutionInfo(); taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0"); taskExecutionInfo1.setStartTime(System.currentTimeMillis()); taskExecutionInfo1.setState(TaskStateEnum.PENDING); taskExecutionInfo1.setLowWatermark(0L); taskExecutionInfo1.setHighWatermark(1000L); Table table1 = new Table(); table1.setNamespace("Test"); table1.setName("Test1"); table1.setType(TableTypeEnum.SNAPSHOT_ONLY); taskExecutionInfo1.setTable(table1); MetricArray taskMetrics1 = new MetricArray(); Metric taskMetric1 = new Metric(); taskMetric1.setValue("100"); taskMetrics1.add(taskMetric1); taskExecutionInfo1.setMetrics(taskMetrics1); Map<String, String> taskProperties1 = Maps.newHashMap(); taskProperties1.put("k1", "v1"); taskExecutionInfo1.setTaskProperties(new StringMap(taskProperties1)); taskExecutionInfos.add(taskExecutionInfo1); TaskExecutionInfo taskExecutionInfo2 = new TaskExecutionInfo(); taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1"); taskExecutionInfo2.setStartTime(System.currentTimeMillis()); taskExecutionInfo2.setState(TaskStateEnum.PENDING); taskExecutionInfo2.setLowWatermark(0L);
@Test(dependsOnMethods = {"testSetAndGet"}) public void testToTaskExecutionInfo() { TaskExecutionInfo taskExecutionInfo = this.taskState.toTaskExecutionInfo(); Assert.assertEquals(taskExecutionInfo.getJobId(), "Job-1"); Assert.assertEquals(taskExecutionInfo.getTaskId(), "Task-1"); Assert.assertEquals(taskExecutionInfo.getHighWatermark().longValue(), 2000L); Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000L); Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name()); Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar"); } }
@Test(dependsOnMethods = {"testInsert"}) public void testUpdate() throws IOException { for (JobExecutionInfo jobExecutionInfo : this.expectedJobExecutionInfos) { jobExecutionInfo.setEndTime(System.currentTimeMillis()); jobExecutionInfo.setDuration(jobExecutionInfo.getEndTime() - jobExecutionInfo.getStartTime()); jobExecutionInfo.setState(JobStateEnum.COMMITTED); jobExecutionInfo.setCompletedTasks(jobExecutionInfo.getLaunchedTasks()); for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) { taskExecutionInfo.setEndTime(jobExecutionInfo.getEndTime()); taskExecutionInfo.setDuration(taskExecutionInfo.getEndTime() - taskExecutionInfo.getStartTime()); taskExecutionInfo.setState(TaskStateEnum.COMMITTED); } this.jobHistoryStore.put(jobExecutionInfo); } }
try (PreparedStatement taskMetricQueryStatement = connection .prepareStatement(TASK_METRIC_QUERY_STATEMENT_TEMPLATE)) { taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId()); try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) { MetricArray taskMetrics = new MetricArray(); taskExecutionInfo.setMetrics(taskMetrics); taskPropertiesQueryStatement.setString(1, taskExecutionInfo.getTaskId()); try (ResultSet taskPropertiesRs = taskPropertiesQueryStatement.executeQuery()) { Map<String, String> taskProperties = Maps.newHashMap(); taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
private static void updateTaskExecutionInfo(Connection connection, TaskExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasTaskId()); try (PreparedStatement updateStatement = connection.prepareStatement(TASK_EXECUTION_UPDATE_STATEMENT_TEMPLATE)) { int index = 0; updateStatement .setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); updateStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); updateStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); updateStatement.setString(++index, info.hasState() ? info.getState().name() : null); updateStatement.setString(++index, info.hasFailureException() ? info.getFailureException() : null); updateStatement.setLong(++index, info.hasLowWatermark() ? info.getLowWatermark() : -1); updateStatement.setLong(++index, info.hasHighWatermark() ? info.getHighWatermark() : -1); updateStatement.setString(++index, info.hasTable() && info.getTable().hasNamespace() ? info.getTable().getNamespace() : null); updateStatement .setString(++index, info.hasTable() && info.getTable().hasName() ? info.getTable().getName() : null); updateStatement .setString(++index, info.hasTable() && info.getTable().hasType() ? info.getTable().getType().name() : null); updateStatement.setString(++index, info.getTaskId()); updateStatement.executeUpdate(); } }
TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo(); taskExecutionInfo.setJobId(this.jobId); taskExecutionInfo.setTaskId(this.taskId); if (this.startTime > 0) { taskExecutionInfo.setStartTime(this.startTime); taskExecutionInfo.setEndTime(this.endTime); taskExecutionInfo.setDuration(this.duration); taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name())); if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)); taskExecutionInfo.setHighWatermark(this.getHighWaterMark()); table.setType(TableTypeEnum.valueOf(extract.getType().name())); taskExecutionInfo.setTable(table); taskExecutionInfo.setMetrics(metricArray); taskProperties.put(name, value); taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
TaskExecutionInfo taskExecutionInfo1 = new TaskExecutionInfo(); taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0"); taskExecutionInfo1.setStartTime(System.currentTimeMillis()); taskExecutionInfo1.setState(TaskStateEnum.PENDING); taskExecutionInfo1.setLowWatermark(0L); taskExecutionInfo1.setHighWatermark(1000L); Table table1 = new Table(); table1.setNamespace("Test"); table1.setName("Test1"); table1.setType(TableTypeEnum.SNAPSHOT_ONLY); taskExecutionInfo1.setTable(table1); MetricArray taskMetrics1 = new MetricArray(); Metric taskMetric1 = new Metric(); taskMetric1.setValue("100"); taskMetrics1.add(taskMetric1); taskExecutionInfo1.setMetrics(taskMetrics1); Map<String, String> taskProperties1 = Maps.newHashMap(); taskProperties1.put("k1" + index, "v1" + index); taskExecutionInfo1.setTaskProperties(new StringMap(taskProperties1)); taskExecutionInfos.add(taskExecutionInfo1); TaskExecutionInfo taskExecutionInfo2 = new TaskExecutionInfo(); taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId()); taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1"); taskExecutionInfo2.setStartTime(System.currentTimeMillis()); taskExecutionInfo2.setState(TaskStateEnum.PENDING); taskExecutionInfo2.setLowWatermark(0L);
private void assertTaskExecution(TaskExecutionInfo actual, TaskExecutionInfo expected) { Assert.assertEquals(actual.getJobId(), expected.getJobId()); Assert.assertEquals(actual.getTaskId(), expected.getTaskId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), 0L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLowWatermark(), expected.getLowWatermark()); Assert.assertEquals(actual.getHighWatermark(), expected.getHighWatermark()); Assert.assertEquals(actual.getTable(), expected.getTable()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getTaskProperties(), expected.getTaskProperties()); }
@Test(dependsOnMethods = {"testSetAndGet"}) public void testToJobExecutionInfo() { JobExecutionInfo jobExecutionInfo = this.jobState.toJobExecutionInfo(); Assert.assertEquals(jobExecutionInfo.getJobName(), "TestJob"); Assert.assertEquals(jobExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(jobExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(jobExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(jobExecutionInfo.getDuration().longValue(), 1000L); Assert.assertEquals(jobExecutionInfo.getState().name(), JobState.RunningState.COMMITTED.name()); Assert.assertEquals(jobExecutionInfo.getLaunchedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getCompletedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getJobProperties().get("foo"), "bar"); List<String> taskStateIds = Lists.newArrayList(); for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) { Assert.assertEquals(taskExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000); Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name()); Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar"); taskStateIds.add(taskExecutionInfo.getTaskId()); } Collections.sort(taskStateIds); Assert.assertEquals(taskStateIds, Lists.newArrayList("TestTask-0", "TestTask-1", "TestTask-2")); } }
try (PreparedStatement taskMetricQueryStatement = connection .prepareStatement(TASK_METRIC_QUERY_STATEMENT_TEMPLATE)) { taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId()); try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) { MetricArray taskMetrics = new MetricArray(); taskExecutionInfo.setMetrics(taskMetrics); taskPropertiesQueryStatement.setString(1, taskExecutionInfo.getTaskId()); try (ResultSet taskPropertiesRs = taskPropertiesQueryStatement.executeQuery()) { Map<String, String> taskProperties = Maps.newHashMap(); taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
private static void insertTaskExecutionInfo(Connection connection, TaskExecutionInfo info) throws SQLException { Preconditions.checkArgument(info.hasTaskId()); Preconditions.checkArgument(info.hasJobId()); try (PreparedStatement insertStatement = connection.prepareStatement(TASK_EXECUTION_INSERT_STATEMENT_TEMPLATE)) { int index = 0; insertStatement.setString(++index, info.getTaskId()); insertStatement.setString(++index, info.getJobId()); insertStatement .setTimestamp(++index, info.hasStartTime() ? new Timestamp(info.getStartTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); insertStatement.setTimestamp(++index, info.hasEndTime() ? new Timestamp(info.getEndTime()) : DEFAULT_TIMESTAMP, getCalendarUTCInstance()); insertStatement.setLong(++index, info.hasDuration() ? info.getDuration() : -1); insertStatement.setString(++index, info.hasState() ? info.getState().name() : null); insertStatement.setString(++index, info.hasFailureException() ? info.getFailureException() : null); insertStatement.setLong(++index, info.hasLowWatermark() ? info.getLowWatermark() : -1); insertStatement.setLong(++index, info.hasHighWatermark() ? info.getHighWatermark() : -1); insertStatement.setString(++index, info.hasTable() && info.getTable().hasNamespace() ? info.getTable().getNamespace() : null); insertStatement .setString(++index, info.hasTable() && info.getTable().hasName() ? info.getTable().getName() : null); insertStatement .setString(++index, info.hasTable() && info.getTable().hasType() ? info.getTable().getType().name() : null); insertStatement.executeUpdate(); } }
private TaskExecutionInfo resultSetToTaskExecutionInfo(ResultSet rs) throws SQLException { TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo(); taskExecutionInfo.setTaskId(rs.getString("task_id")); taskExecutionInfo.setJobId(rs.getString("job_id")); try { Timestamp startTime = rs.getTimestamp("start_time"); if (startTime != null) { taskExecutionInfo.setStartTime(startTime.getTime()); taskExecutionInfo.setStartTime(0); taskExecutionInfo.setEndTime(endTime.getTime()); taskExecutionInfo.setEndTime(0); taskExecutionInfo.setDuration(rs.getLong("duration")); String state = rs.getString("state"); if (!Strings.isNullOrEmpty(state)) { taskExecutionInfo.setState(TaskStateEnum.valueOf(state)); taskExecutionInfo.setFailureException(failureException); taskExecutionInfo.setLowWatermark(rs.getLong("low_watermark")); taskExecutionInfo.setHighWatermark(rs.getLong("high_watermark")); Table table = new Table(); String namespace = rs.getString("table_namespace"); taskExecutionInfo.setTable(table);
String taskId = taskPropertiesRs.getString("task_id"); TaskExecutionInfo taskExecutionInfo = taskExecutionInfos.get(jobId).get(taskId); StringMap taskProperties = taskExecutionInfo.getTaskProperties(GetMode.NULL); if (taskProperties == null) { taskProperties = new StringMap(); taskExecutionInfo.setTaskProperties(taskProperties);
String taskId = taskMetricRs.getString("task_id"); TaskExecutionInfo taskExecutionInfo = taskExecutionInfos.get(jobId).get(taskId); MetricArray metricsArray = taskExecutionInfo.getMetrics(GetMode.NULL); if (metricsArray == null) { metricsArray = new MetricArray(); taskExecutionInfo.setMetrics(metricsArray);