@Autowired public MetricsCollector(DataManager dataManager, JsonDB jsonDB, KubernetesClient kubernetes) { this.dataManager = dataManager; this.kubernetes = kubernetes; this.rmh = new JsonDBRawMetrics(jsonDB); this.imh = new IntegrationMetricsHandler(dataManager); }
/** * If Integrations get deleted we should also delete their metrics * * @param activeIntegrationIds * @throws IOException * @throws JsonMappingException */ @Override public void curate(Set<String> activeIntegrationIds) throws IOException, JsonMappingException { //1. Loop over all RawMetrics String json = jsonDB.getAsString(path(), new GetOptions().depth(1)); if (json != null) { Map<String,Boolean> metricsMap = Json.reader().forType(TYPE_REFERENCE).readValue(json); Set<String> rawIntegrationIds = metricsMap.keySet(); for (String rawIntId : rawIntegrationIds) { if (! activeIntegrationIds.contains(rawIntId)) { jsonDB.delete(path(rawIntId)); } } } }
@Test public void testGetRawMetrics() throws IOException { MetricsCollector collector = new MetricsCollector(null, jsondb, null); Map<String,RawMetrics> metrics = jsondbRM.getRawMetrics("intId1"); assertThat(metrics.size()).isEqualTo(3); assertThat(metrics.keySet()).contains("HISTORY1"); //let's kill pod2, this so add pod2's metrics to the history Set<String> livePodIds = new HashSet<>(Arrays.asList("pod1")); jsondbRM.curate("intId1", metrics, livePodIds); Map<String,RawMetrics> metrics2 = jsondbRM.getRawMetrics("intId1"); assertThat(metrics2.size()).isEqualTo(2); assertThat(metrics2.keySet()).contains("HISTORY1"); collector.close(); }
@Test public void testDeadPodCurator() throws IOException, ParseException { String integrationId = "intId1"; MetricsCollector collector = new MetricsCollector(null, jsondb, null); //Update pod1 metrics and kill pod1 Set<String> livePodIds = new HashSet<String>( Arrays.asList("pod2", "pod3", "pod4", "pod5")); jsondb.update(JsonDBRawMetrics.path("intId1","pod1"), Json.writer().writeValueAsString(raw("intId1","1","pod1",12L,"31-01-2018 10:22:56"))); Map<String,RawMetrics> metrics = jsondbRM.getRawMetrics(integrationId); IntegrationMetricsSummary summary = intMH .compute(integrationId, metrics, livePodIds); assertThat(summary.getMessages()).isEqualTo(18); assertThat(summary.getErrors()).isEqualTo(3); //Oldest living pod is now pod2 assertThat(summary.getStart().get()).isEqualTo(sdf.parse("31-01-2018 10:22:56")); collector.close(); }
private void load() throws IOException, ParseException { jsondbRM.persist(raw("intId1","1","pod1",3L, "31-01-2018 10:20:56")); jsondbRM.persist(raw("intId1","1","pod2",3L, "31-01-2018 10:22:56")); jsondbRM.persist(raw("intId1","1","HISTORY1",3L, "22-01-2015 10:20:56")); jsondbRM.persist(raw("intId2","1","pod3",3L, "31-01-2018 10:20:56")); jsondbRM.persist(raw("intId3","1","pod4",3L, "31-01-2018 10:20:56")); jsondbRM.persist(raw("intId3","1","pod5",3L, "31-01-2018 10:20:56")); }
@Test public void testGetIntegrationSummary() throws IOException, ParseException { String integrationId = "intId1"; Set<String> livePodIds = new HashSet<String>( Arrays.asList("pod1", "pod2", "pod3", "pod4", "pod5")); MetricsCollector collector = new MetricsCollector(null, jsondb, null); Map<String,RawMetrics> metrics = jsondbRM.getRawMetrics(integrationId); IntegrationMetricsSummary summary = intMH .compute(integrationId, metrics, livePodIds); assertThat(summary.getMessages()).isEqualTo(9); assertThat(summary.getErrors()).isEqualTo(3); //Oldest living pod assertThat(summary.getStart().get()).isEqualTo(sdf.parse("31-01-2018 10:20:56")); //Update pod2, add 6 messages jsondb.update(JsonDBRawMetrics.path("intId1","pod2"), Json.writer().writeValueAsString(raw("intId1","2","pod2",9L,"31-01-2018 10:22:56"))); Map<String,RawMetrics> updatedMetrics = jsondbRM.getRawMetrics(integrationId); IntegrationMetricsSummary updatedSummary = intMH .compute(integrationId, updatedMetrics, livePodIds); assertThat(updatedSummary.getMessages()).isEqualTo(15); assertThat(updatedSummary.getErrors()).isEqualTo(3); collector.close(); }
/** * Obtains all RawMetrics entries in the DB for the current integration * * @param integrationId - the integrationId for which we are obtaining the metrics * @return a Map containing all RawMetrics entries for the current integration, * the key is either HISTORY or the podName. * @throws IOException */ @Override public Map<String,RawMetrics> getRawMetrics(String integrationId) throws IOException { //try to obtain all raw metrics in this integration Map<String,RawMetrics> metrics = new HashMap<>(); String path = path(integrationId); String json = jsonDB.getAsString(path); if (json != null) { metrics = Json.reader().forType(VALUE_TYPE_REF).readValue(json); } return metrics; }
@Test public void testDeletedIntegrationsCurator() throws IOException, ParseException { String integrationId = "intId1"; Set<String> livePodIds = new HashSet<String>( Arrays.asList("pod1", "pod2")); Map<String,RawMetrics> metrics = jsondbRM.getRawMetrics(integrationId); IntegrationMetricsSummary summary = intMH.compute(integrationId, metrics, livePodIds); dataManager.create(summary); assertThat(metrics.size()).isEqualTo(3); assertThat(dataManager.fetchAll(IntegrationMetricsSummary.class).getTotalCount()).isEqualTo(1); //Now pretend to delete the integration itself and //run the curator with no active integrations jsondbRM.curate(new HashSet<String>()); intMH.curate(new HashSet<String>()); //expect all metrics to be deleted Map<String,RawMetrics> metricsAfter = jsondbRM.getRawMetrics(integrationId); assertThat(metricsAfter.size()).isEqualTo(0); assertThat(dataManager.fetchAll(IntegrationMetricsSummary.class).getTotalCount()).isEqualTo(0); } }
@Before public void before() throws IOException, ParseException { JdbcDataSource ds = new JdbcDataSource(); ds.setURL("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL"); DBI dbi = new DBI(ds); this.jsondb = new SqlJsonDB(dbi, null, Arrays.asList(new Index("/pair", "key")) ); try { this.jsondb.dropTables(); } catch (Exception e) { } this.jsondb.createTables(); jsondbRM = new JsonDBRawMetrics(jsondb); load(); CacheManager cacheManager = new LRUCacheManager(100); EncryptionComponent encryptionComponent = new EncryptionComponent(null); ResourceLoader resourceLoader = new DefaultResourceLoader(); //Create Data Manager dataManager = new DataManager(cacheManager, Collections.emptyList(), null, encryptionComponent, resourceLoader); intMH = new IntegrationMetricsHandler(dataManager); }
/** * Persists the latest metrics of a live pod to the database. */ @Override public void persist(RawMetrics rawMetrics) { try { //persist the latest rawMetrics String path = path(rawMetrics.getIntegrationId(), rawMetrics.getPod()); String json = Json.writer().writeValueAsString(rawMetrics); if (jsonDB.exists(path)) { //only update if not the same (don't cause unnecessary and expensive writes) if (! jsonDB.getAsString(path).equals(json)) { jsonDB.update(path, json); } } else { jsonDB.set(path, json); } } catch (JsonProcessingException e) { LOGGER.error("Error persisting metrics!", e); } }
@Test public void testGetMetricsForIntegration1() throws IOException { String json = jsondb.getAsString(JsonDBRawMetrics.path("intId1"), new GetOptions().prettyPrint(true)); Map<String,RawMetrics> metrics = Json.reader().forType(new TypeReference<Map<String,RawMetrics>>() {}).readValue(json); assertThat(metrics.size()).isEqualTo(3); assertThat(metrics.keySet()).contains("HISTORY1"); }