private void addMetricsToJobExecutions(Connection connection, JobExecutionQuery query, Map<String, JobExecutionInfo> jobExecutionInfos) throws SQLException { if (query.isIncludeJobMetrics() && jobExecutionInfos.size() > 0) { String template = String.format(JOB_METRIC_QUERY_STATEMENT_TEMPLATE, getInPredicate(jobExecutionInfos.size())); int index = 1; try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(template)) { for (String jobId : jobExecutionInfos.keySet()) { jobMetricQueryStatement.setString(index++, jobId); } try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { while (jobMetricRs.next()) { String jobId = jobMetricRs.getString("job_id"); JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId); MetricArray metricArray = jobExecutionInfo.getMetrics(GetMode.NULL); if (metricArray == null) { metricArray = new MetricArray(); jobExecutionInfo.setMetrics(metricArray); } metricArray.add(resultSetToMetric(jobMetricRs)); } } } } }
private void assertTaskExecution(TaskExecutionInfo actual, TaskExecutionInfo expected) { Assert.assertEquals(actual.getJobId(), expected.getJobId()); Assert.assertEquals(actual.getTaskId(), expected.getTaskId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), 0L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLowWatermark(), expected.getLowWatermark()); Assert.assertEquals(actual.getHighWatermark(), expected.getHighWatermark()); Assert.assertEquals(actual.getTable(), expected.getTable()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getTaskProperties(), expected.getTaskProperties()); }
/** * Print out various metrics * @param metrics Metrics to print */ private static void printMetrics(MetricArray metrics) { System.out.println(); if (metrics.size() == 0) { System.out.println("No metrics found."); return; } List<List<String>> data = new ArrayList<>(); List<String> flags = Arrays.asList("", "-"); for (Metric metric : metrics) { data.add(Arrays.asList(metric.getName(), metric.getValue())); } new CliTablePrinter.Builder() .data(data) .flags(flags) .delimiterWidth(2) .build() .printTable(); }
public MetricArray(Collection<Metric> c) { this(new DataList(c.size())); addAll(c); }
/** * Print out various metrics * @param metrics Metrics to print */ private static void printMetrics(MetricArray metrics) { System.out.println(); if (metrics.size() == 0) { System.out.println("No metrics found."); return; } List<List<String>> data = new ArrayList<>(); List<String> flags = Arrays.asList("", "-"); for (Metric metric : metrics) { data.add(Arrays.asList(metric.getName(), metric.getValue())); } new CliTablePrinter.Builder() .data(data) .flags(flags) .delimiterWidth(2) .build() .printTable(); }
MetricArray metricsArray = taskExecutionInfo.getMetrics(GetMode.NULL); if (metricsArray == null) { metricsArray = new MetricArray(); taskExecutionInfo.setMetrics(metricsArray); metricsArray.add(resultSetToMetric(taskMetricRs));
private static void assertTaskExecution(TaskExecutionInfo actual, TaskExecutionInfo expected) { Assert.assertEquals(actual.getJobId(), expected.getJobId()); Assert.assertEquals(actual.getTaskId(), expected.getTaskId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), -1L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLowWatermark(), expected.getLowWatermark()); Assert.assertEquals(actual.getHighWatermark(), expected.getHighWatermark()); Assert.assertEquals(actual.getTable(), expected.getTable()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getTaskProperties(), expected.getTaskProperties()); }
jobMetricQueryStatement.setString(1, jobRs.getString(2)); try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { MetricArray jobMetrics = new MetricArray(); while (jobMetricRs.next()) { jobMetrics.add(resultSetToMetric(jobMetricRs)); taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId()); try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) { MetricArray taskMetrics = new MetricArray(); while (taskMetricRs.next()) { taskMetrics.add(resultSetToMetric(taskMetricRs));
private static void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), -1L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
MetricArray metricArray = new MetricArray(); counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name())); counter.setValue(Long.toString(((Counter) entry.getValue()).getCount())); metricArray.add(counter); meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name())); meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate())); metricArray.add(meter); gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name())); gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString()); metricArray.add(gauge);
private void assertJobExecution(JobExecutionInfo actual, JobExecutionInfo expected) { Assert.assertEquals(actual.getJobName(), expected.getJobName()); Assert.assertEquals(actual.getJobId(), expected.getJobId()); if (expected.hasDuration()) { Assert.assertEquals(actual.getDuration(), expected.getDuration()); } else { Assert.assertEquals(actual.getDuration().longValue(), 0L); } Assert.assertEquals(actual.getState(), expected.getState()); Assert.assertEquals(actual.getLaunchedTasks(), expected.getLaunchedTasks()); Assert.assertEquals(actual.getCompletedTasks(), expected.getCompletedTasks()); Assert.assertEquals(actual.getLauncherType(), expected.getLauncherType()); Assert.assertEquals(actual.getTrackingUrl(), expected.getTrackingUrl()); Assert.assertEquals(actual.getMetrics(), expected.getMetrics()); for (int i = 0; i < actual.getMetrics().size(); i++) { assertMetric(actual.getMetrics().get(i), expected.getMetrics().get(i)); } Assert.assertEquals(actual.getJobProperties(), expected.getJobProperties()); Assert.assertEquals(actual.getTaskExecutions().size(), expected.getTaskExecutions().size()); for (int i = 0; i < actual.getTaskExecutions().size(); i++) { assertTaskExecution(actual.getTaskExecutions().get(i), expected.getTaskExecutions().get(i)); } }
MetricArray metricArray = new MetricArray(); counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name())); counter.setValue(Long.toString(((Counter) entry.getValue()).getCount())); metricArray.add(counter); meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name())); meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate())); metricArray.add(meter); gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name())); gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString()); metricArray.add(gauge);
MetricArray jobMetrics = new MetricArray(); Metric jobMetric1 = new Metric(); jobMetric1.setGroup("JOB"); jobMetric1.setType(MetricTypeEnum.COUNTER); jobMetric1.setValue("100"); jobMetrics.add(jobMetric1); jobExecutionInfo.setMetrics(jobMetrics); table1.setType(TableTypeEnum.SNAPSHOT_ONLY); taskExecutionInfo1.setTable(table1); MetricArray taskMetrics1 = new MetricArray(); Metric taskMetric1 = new Metric(); taskMetric1.setGroup("TASK"); taskMetric1.setType(MetricTypeEnum.COUNTER); taskMetric1.setValue("100"); taskMetrics1.add(taskMetric1); taskExecutionInfo1.setMetrics(taskMetrics1); Map<String, String> taskProperties1 = Maps.newHashMap(); table2.setType(differentTableType ? TableTypeEnum.SNAPSHOT_APPEND : TableTypeEnum.SNAPSHOT_ONLY); taskExecutionInfo2.setTable(table2); MetricArray taskMetrics2 = new MetricArray(); Metric taskMetric2 = new Metric(); taskMetric2.setGroup("TASK"); taskMetric2.setType(MetricTypeEnum.COUNTER); taskMetric2.setValue("100");
MetricArray jobMetrics = new MetricArray(); Metric jobMetric1 = new Metric(); jobMetric1.setGroup("JOB"); jobMetric1.setType(MetricTypeEnum.COUNTER); jobMetric1.setValue("100"); jobMetrics.add(jobMetric1); jobExecutionInfo.setMetrics(jobMetrics); table1.setType(TableTypeEnum.SNAPSHOT_ONLY); taskExecutionInfo1.setTable(table1); MetricArray taskMetrics1 = new MetricArray(); Metric taskMetric1 = new Metric(); taskMetric1.setGroup("TASK"); taskMetric1.setType(MetricTypeEnum.COUNTER); taskMetric1.setValue("100"); taskMetrics1.add(taskMetric1); taskExecutionInfo1.setMetrics(taskMetrics1); Map<String, String> taskProperties1 = Maps.newHashMap(); table2.setType(TableTypeEnum.SNAPSHOT_ONLY); taskExecutionInfo2.setTable(table2); MetricArray taskMetrics2 = new MetricArray(); Metric taskMetric2 = new Metric(); taskMetric2.setGroup("TASK"); taskMetric2.setType(MetricTypeEnum.COUNTER); taskMetric2.setValue("100");
private void addMetricsToJobExecutions(Connection connection, JobExecutionQuery query, Map<String, JobExecutionInfo> jobExecutionInfos) throws SQLException { if (query.isIncludeJobMetrics() && jobExecutionInfos.size() > 0) { String template = String.format(JOB_METRIC_QUERY_STATEMENT_TEMPLATE, getInPredicate(jobExecutionInfos.size())); int index = 1; try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(template)) { for (String jobId : jobExecutionInfos.keySet()) { jobMetricQueryStatement.setString(index++, jobId); } try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { while (jobMetricRs.next()) { String jobId = jobMetricRs.getString("job_id"); JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId); MetricArray metricArray = jobExecutionInfo.getMetrics(GetMode.NULL); if (metricArray == null) { metricArray = new MetricArray(); jobExecutionInfo.setMetrics(metricArray); } metricArray.add(resultSetToMetric(jobMetricRs)); } } } } }
MetricArray metricsArray = taskExecutionInfo.getMetrics(GetMode.NULL); if (metricsArray == null) { metricsArray = new MetricArray(); taskExecutionInfo.setMetrics(metricsArray); metricsArray.add(resultSetToMetric(taskMetricRs));
jobMetricQueryStatement.setString(1, jobRs.getString(2)); try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) { MetricArray jobMetrics = new MetricArray(); while (jobMetricRs.next()) { jobMetrics.add(resultSetToMetric(jobMetricRs)); taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId()); try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) { MetricArray taskMetrics = new MetricArray(); while (taskMetricRs.next()) { taskMetrics.add(resultSetToMetric(taskMetricRs));
MetricArray metricArray = new MetricArray(); counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name())); counter.setValue(Long.toString(((Counter) entry.getValue()).getCount())); metricArray.add(counter); meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name())); meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate())); metricArray.add(meter); gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name())); gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString()); metricArray.add(gauge);
MetricArray metricArray = new MetricArray(); counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name())); counter.setValue(Long.toString(((Counter) entry.getValue()).getCount())); metricArray.add(counter); meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name())); meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate())); metricArray.add(meter); gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name())); gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString()); metricArray.add(gauge);