private void upsertJobMetrics(Connection connection, JobExecutionInfo jobExecutionInfo) throws SQLException { if (jobExecutionInfo.hasMetrics()) { Optional<PreparedStatement> upsertStatement = Optional.absent(); int batchSize = 0; for (Metric metric : jobExecutionInfo.getMetrics()) { if (!upsertStatement.isPresent()) { upsertStatement = Optional.of(connection.prepareStatement(JOB_METRIC_UPSERT_STATEMENT_TEMPLATE)); } addMetricToBatch(upsertStatement.get(), metric, jobExecutionInfo.getJobId()); if (batchSize++ > 1000) { executeBatches(upsertStatement); upsertStatement = Optional.absent(); batchSize = 0; } } executeBatches(upsertStatement); } }
private void addMetricsToJobExecutions(Connection connection, JobExecutionQuery query, Map<String, JobExecutionInfo> jobExecutionInfos) throws SQLException { if (query.isIncludeJobMetrics() && jobExecutionInfos.size() > 0) { String template = String.format(JOB_METRIC_QUERY_STATEMENT_TEMPLATE, getInPredicate(jobExecutionInfos.size())); int index = 1; try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(template)) { for (String jobId : jobExecutionInfos.keySet()) { jobMetricQueryStatement.setString(index++, jobId); } try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { while (jobMetricRs.next()) { String jobId = jobMetricRs.getString("job_id"); JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId); MetricArray metricArray = jobExecutionInfo.getMetrics(GetMode.NULL); if (metricArray == null) { metricArray = new MetricArray(); jobExecutionInfo.setMetrics(metricArray); } metricArray.add(resultSetToMetric(jobMetricRs)); } } } } }
.printTable(); JobInfoPrintUtils.printMetrics(jobExecutionInfo.getMetrics());
MetricArray metrics = jobInfo.getMetrics(); Double recordsProcessed = null; Double recordsFailed = null;
for (Metric metric : jobExecutionInfo.getMetrics()) { boolean insert = !existsMetric(connection, JOB_METRIC_EXIST_QUERY_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), metric);
private static void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), -1L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
private void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), 0L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getLauncherType(), expected.getLauncherType()); Assert.assertEquals(actual.getTrackingUrl(), expected.getTrackingUrl()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
private void addMetricsToJobExecutions(Connection connection, JobExecutionQuery query, Map<String, JobExecutionInfo> jobExecutionInfos) throws SQLException { if (query.isIncludeJobMetrics() && jobExecutionInfos.size() > 0) { String template = String.format(JOB_METRIC_QUERY_STATEMENT_TEMPLATE, getInPredicate(jobExecutionInfos.size())); int index = 1; try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(template)) { for (String jobId : jobExecutionInfos.keySet()) { jobMetricQueryStatement.setString(index++, jobId); } try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { while (jobMetricRs.next()) { String jobId = jobMetricRs.getString("job_id"); JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId); MetricArray metricArray = jobExecutionInfo.getMetrics(GetMode.NULL); if (metricArray == null) { metricArray = new MetricArray(); jobExecutionInfo.setMetrics(metricArray); } metricArray.add(resultSetToMetric(jobMetricRs)); } } } } }
private void upsertJobMetrics(Connection connection, JobExecutionInfo jobExecutionInfo) throws SQLException { if (jobExecutionInfo.hasMetrics()) { Optional<PreparedStatement> upsertStatement = Optional.absent(); int batchSize = 0; for (Metric metric : jobExecutionInfo.getMetrics()) { if (!upsertStatement.isPresent()) { upsertStatement = Optional.of(connection.prepareStatement(JOB_METRIC_UPSERT_STATEMENT_TEMPLATE)); } addMetricToBatch(upsertStatement.get(), metric, jobExecutionInfo.getJobId()); if (batchSize++ > 1000) { executeBatches(upsertStatement); upsertStatement = Optional.absent(); batchSize = 0; } } executeBatches(upsertStatement); } }
.printTable(); JobInfoPrintUtils.printMetrics(jobExecutionInfo.getMetrics());
MetricArray metrics = jobInfo.getMetrics(); Double recordsProcessed = null; Double recordsFailed = null;
for (Metric metric : jobExecutionInfo.getMetrics()) { boolean insert = !existsMetric(connection, JOB_METRIC_EXIST_QUERY_STATEMENT_TEMPLATE, jobExecutionInfo.getJobId(), metric);