@Override public void setup(String dirName) { defaultStorage = new GzipStorage(); defaultStorage.setup(dirName); dailyStorage = new GzipStorage(); dailyStorage.setup(dirName + "/day"); weeklyStorage = new GzipStorage(); weeklyStorage.setup(dirName + "/week"); monthlyStorage = new GzipStorage(); monthlyStorage.setup(dirName + "/month"); } }
@Test public void shouldLoadDetailsFromDefaultStorage(){ //given Date from = parse("2016-10-01 00:00:00"); Date to = parse("2016-10-01 23:58:00"); //when storage.loadDetails("detailsId", from, to, "instance", "type"); //then verify(defaultStorage).loadDetails(eq("detailsId"), eq(from), eq(to), eq("instance"), eq("type")); verifyNoMoreInteractions(dailyStorage); verifyNoMoreInteractions(weeklyStorage); verifyNoMoreInteractions(monthlyStorage); }
@Test public void should_load_histogram_by_instance_and_type() throws IOException{ //given Stats stats = stats(dt(year+"-01-01 08:05:00"), dt(year+"-01-01 08:15:00"), stat(100)); //save the same but with different instance and type storage.save(stats, "i1", null); storage.save(stats, "i1", "t1"); storage.save(stats, "i2", "t1"); //when Histogram histogramI = storage.loadHistogram(STAT_NAME, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:30:00"), "i1", null, null); Histogram histogramT = storage.loadHistogram(STAT_NAME, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:30:00"), null, "t1", null); Histogram histogramIT = storage.loadHistogram(STAT_NAME, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:30:00"), "i1", "t1", null); //then assertEquals( 2, histogramI.getHistogram().size()); assertEquals( 2, histogramT.getHistogram().size()); assertEquals( 1, histogramIT.getHistogram().size()); }
@Test public void should_find_one_stats_in_exact_range() throws IOException{ String id1 = storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:14:59")), null, null); String id2 = storage.save(stats(dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:30:00")), null, null); String id3 = storage.save(stats(dt(year+"-01-01 08:30:01"), dt(year+"-01-01 08:45:00")), null, null); List<String> ids = storage.find(dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:30:00"), null, null); assertEquals(1, ids.size() ); assertEquals(id2, ids.get(0) ); }
@Test public void should_load_top_level() throws IOException{ //given Stat parentStat = stat(100); Stat childStat = stat(10); parentStat.ensureChildrenMap(); parentStat.getChildren().put("child", childStat); Stats stats = stats(dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:30:00"), parentStat); storage.save(stats, null, null); //when Stats topLevel = storage.loadAggregated(true, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:45:00"), null, null); //then Stat root = topLevel.getMap().get(STAT_NAME); assertNotNull(root); assertNotNull( root.getChildren() ); //not null map means that root has children ... assertEquals( 0, root.getChildren().size());///but for top level only children are empty }
@Test public void should_load_details_by_stats_ids() throws IOException{ //given Stats stats1 = stats( dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00"), stat(100, 200)//hits = 2, avg = 150 ); Stats stats2 = stats( dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:30:00"), stat(1000)//hits = 1, avg = 1000 ); String id1 = storage.save(stats1, null, null); String id2 = storage.save(stats2, null, null); //when StatDetails details = storage.loadDetails(STAT_NAME, asList(id1, id2)); //then Stat expected = stats1.getMap().get(STAT_NAME).deepCopy(); expected.merge(stats2.getMap().get(STAT_NAME)); assertEquals(STAT_NAME, details.getId() ); assertEquals( expected, details.getMerged()); }
@Test public void should_find_filters_for_type() throws IOException{ storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), "i1", "t1"); storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), "i2", "t1"); storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), "i1", "t2"); storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), "i3", "t2"); Filters filters = storage.findFilters(dt(year+"-01-01 07:10:00"), dt(year+"-01-01 08:40:00"), "t1"); assertEquals( 2, filters.getInstances().size() ); assertTrue(filters.getInstances().contains("i1")); assertTrue(filters.getInstances().contains("i2")); assertEquals( 1, filters.getTypes().size() ); assertTrue(filters.getTypes().contains("t1")); }
@Test public void shouldLoadHistogramFromDefaultStorage(){ //given Date from = parse("2016-10-01 00:00:00"); Date to = parse("2016-10-01 23:58:00"); //when storage.loadHistogram("detailsId", from, to, "instance", "type", null); //then verify(defaultStorage).loadHistogram(eq("detailsId"), eq(from), eq(to), eq("instance"), eq("type"), isNull(String.class)); verifyNoMoreInteractions(dailyStorage); verifyNoMoreInteractions(weeklyStorage); verifyNoMoreInteractions(monthlyStorage); }
@Test public void should_save_and_load_single_stats() throws IOException{ Stats stats = stats( dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")); String identifier = storage.save(stats, null, null); Stats loaded = storage.load(identifier); //we don't compare implementation specific data loaded.setInfo(stats.getInfo()); assertEquals(stats, loaded); assertEquals(jsonUtil.toJson(stats), jsonUtil.toJson(loaded)); }
/** * @param stats * @return true if successfully saved stats - otherwise false */ public boolean safeSave(Stats stats){ if( storage == null ){ return false; } try{ ensureDatesAreSet(stats); long start = System.currentTimeMillis(); storage.save(stats, appInstance, appType); lastSave.set(dateUtil.now().getTime()); lastError.set(0); log.info("Saved stats in {}ms", System.currentTimeMillis() - start); return true; }catch(Exception e){ Throwable cause = firstNonNull(getRootCause(e), e); log.warn("Failed to save stats - next attempt in 5 minutes. Error type: {}, Message: {}", cause.getClass().getName(), cause.getMessage()); log.debug("Failed to save stats - next attempt in 5 minutes. ", e); lastError.set(dateUtil.now().getTime()); return false; } }
@GET @Path("/data/filters") @Produces("application/json") public Response filters(@QueryParam("from") String from, @QueryParam("to") String to, @QueryParam("past") String past){ TimeRange range = new TimeRange(from, to, past, null, null); Filters filters = null; if( canSearch(range) ){ filters = getStorage().findFilters(range.getFromDate(), range.getToDate(), null); } if( filters == null){ filters = new Filters(null, null); } filters.setInstances(addDefaultAndSort(filters.getInstances(), Properties.getAppInstance())); filters.setTypes( addDefaultAndSort(filters.getTypes(), Properties.getAppType())); return noCacheResponse(jsonUtil.toJson(filters)); }
private Stats topLevelForRange(TimeRange range) { Stats result; if( canSearch(range) ){ result = getStorage().loadAggregated(true, range.getFromDate(), range.getToDate(), range.getInstance(), range.getType()); if( inMemoryInRange(range) ){ Stats current = topLevelInMemory(); result.merge(current, false); result.setInfo("Added in-memory data. Original info: " + result.getInfo()); } } else{ result = topLevelInMemory(); result.setInfo("in-memory data only. Original info: " + result.getInfo()); } return result; }
public void safeCleanup() { if( daysToKeep <= 0 || lastError.get() > 0 ){ return; } try{ long start = System.currentTimeMillis(); storage.cleanup(daysToKeep, appType); lastSave.set(dateUtil.now().getTime()); log.info("Performed cleanup in {}ms", System.currentTimeMillis() - start); }catch(Exception e){ Throwable cause = firstNonNull(getRootCause(e), e); log.warn("Failed to cleanup stats. Error type: {}, Message: {}", cause.getClass().getName(), cause.getMessage()); log.debug("Failed to cleanup stats.", e); } }
@Test public void should_load_aggregated_by_instance() throws IOException{ //given storage.save(stats( dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:10:00"), stat(10)), "i1", null); storage.save(stats( dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:20:00"), stat(100)), "i1", "t1"); storage.save(stats( dt(year+"-01-01 08:30:00"), dt(year+"-01-01 08:45:00"), stat(1000)), "i2", "t1"); //when Stats aggrI = storage.loadAggregated(false, dt(year+"-01-01 07:00:00"), dt(year+"-01-01 09:30:00"), "i1", null); //then Stats aggregated = aggrI; Stat stat = aggregated.getMap().get(STAT_NAME); assertEquals( dt(year+"-01-01 08:00:00"), aggregated.getStartDate()); assertEquals( dt(year+"-01-01 08:20:00"), aggregated.getEndDate()); assertEquals( 1, aggregated.getMap().size()); assertEquals( 55L, stat.getAvg()); assertEquals( 2L, stat.getHits()); assertEquals( 100L, stat.getMax()); assertEquals( 10L, stat.getMin()); }
@Test public void should_load_details_by_instance_and_type() throws IOException{ //given Stats stats1 = stats(dt(year+"-01-01 08:05:00"), dt(year+"-01-01 08:15:00"), stat(2)); Stats stats10 = stats(dt(year+"-01-01 08:05:00"), dt(year+"-01-01 08:15:00"), stat(10)); Stats stats100 = stats(dt(year+"-01-01 08:05:00"), dt(year+"-01-01 08:15:00"), stat(100)); //save the same but with different instance and type storage.save(stats1, "i1", null); storage.save(stats10, "i1", "t1"); storage.save(stats100, "i2", "t1"); //when StatDetails detailsI = storage.loadDetails(STAT_NAME, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:30:00"), "i1", null); StatDetails detailsT = storage.loadDetails(STAT_NAME, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:30:00"), null, "t1"); StatDetails detailsIT = storage.loadDetails(STAT_NAME, dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:30:00"), "i1", "t1"); //then assertEquals( 6, detailsI.getMerged().getAvg()); assertEquals( 55, detailsT.getMerged().getAvg()); assertEquals( 10, detailsIT.getMerged().getAvg()); }
@Test public void should_find_all_stats_in_bigger_range() throws IOException{ String id1 = storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), null, null); String id2 = storage.save(stats(dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:30:00")), null, null); String id3 = storage.save(stats(dt(year+"-01-01 08:30:00"), dt(year+"-01-01 08:45:00")), null, null); List<String> ids = storage.find(dt(year+"-01-01 08:10:00"), dt(year+"-01-01 08:40:00"), null, null); assertEquals(3, ids.size() ); assertTrue(ids.containsAll(asList(id1, id2, id3))); }
@Test public void should_find_filters_for_date_range() throws IOException{ storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), "i1", "t1"); storage.save(stats(dt(year+"-01-01 08:30:00"), dt(year+"-01-01 08:40:00")), "i2", "t1"); storage.save(stats(dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:15:00")), "i1", "t2"); storage.save(stats(dt(year+"-01-01 08:30:00"), dt(year+"-01-01 08:40:00")), "i3", "t2"); Filters filters = storage.findFilters(dt(year+"-01-01 08:35:00"), dt(year+"-01-01 08:37:00"), null); assertEquals( 2, filters.getInstances().size() ); assertTrue(filters.getInstances().contains("i2")); assertTrue(filters.getInstances().contains("i3")); assertEquals( 2, filters.getTypes().size() ); assertTrue(filters.getTypes().contains("t1")); assertTrue(filters.getTypes().contains("t2")); }
@Test public void shouldLoadWeeklyHistogramFromDailyStorage(){ //given Date from = parse("2016-10-01 00:00:00"); Date to = parse("2016-10-08 01:58:00"); //when storage.loadHistogram("detailsId", from, to, "instance", "type", null); //then verifyNoMoreInteractions(defaultStorage); verify(dailyStorage).loadHistogram(eq("detailsId"), eq(from), eq(to), isNull(String.class), eq("type"), isNull(String.class)); verifyNoMoreInteractions(weeklyStorage); verifyNoMoreInteractions(monthlyStorage); }
@Test public void shouldDelegateSaveOfOriginalStats(){ //given Stats stats = new Stats(); stats.setStartDate(parse("2000-03-03 13:15:12")); stats.setEndDate(parse("2000-03-03 13:25:07")); stats.getMap().put("x", Stat.emptyStat()); //when storage.save(stats, "instance", "type"); //then verify(defaultStorage).save(eq(stats), eq("instance"), eq("type")); }
@Test public void should_load_aggregated_by_instance_and_type() throws IOException{ //given storage.save(stats( dt(year+"-01-01 08:00:00"), dt(year+"-01-01 08:10:00"), stat(10)), "i1", null); storage.save(stats( dt(year+"-01-01 08:15:00"), dt(year+"-01-01 08:20:00"), stat(100)), "i1", "t1"); storage.save(stats( dt(year+"-01-01 08:30:00"), dt(year+"-01-01 08:45:00"), stat(1000)), "i2", "t1"); //when Stats aggrIT = storage.loadAggregated(false, dt(year+"-01-01 07:00:00"), dt(year+"-01-01 09:30:00"), "i1", "t1"); //then Stats aggregated = aggrIT; Stat stat = aggregated.getMap().get(STAT_NAME); assertEquals( dt(year+"-01-01 08:15:00"), aggregated.getStartDate()); assertEquals( dt(year+"-01-01 08:20:00"), aggregated.getEndDate()); assertEquals( 1, aggregated.getMap().size()); assertEquals( 100L, stat.getAvg()); assertEquals( 1L, stat.getHits()); assertEquals( 100L, stat.getMax()); assertEquals( 100L, stat.getMin()); } }