@Override public CategoryRecordIdAndMetric getCategoryIdAndMetric(final Integer metricId, final InternalTenantContext context) throws UnableToObtainConnectionException { CategoryRecordIdAndMetric result = metricsCache.get(metricId); if (result == null) { result = delegate.getCategoryIdAndMetric(metricId, context); if (result != null) { synchronized (metricsCache) { if (metricsCache.get(metricId) == null) { metricsCache.put(metricId, result); } } } } return result; }
private List<Integer> translateCategoriesAndMetricNamesToMetricIds(final Map<String, Collection<String>> metricsPerCategory) { final List<Integer> metricIds = new ArrayList<Integer>(metricsPerCategory.keySet().size()); for (final String category : metricsPerCategory.keySet()) { final Integer categoryId = timelineDao.getEventCategoryId(category, context); if (categoryId == null) { // Ignore continue; } for (final String metricName : metricsPerCategory.get(category)) { final Integer sampleKindId = timelineDao.getMetricId(categoryId, metricName, context); if (sampleKindId == null) { // Ignore continue; } metricIds.add(sampleKindId); } } return metricIds; }
@Override public BiMap<Integer, CategoryRecordIdAndMetric> getMetrics(final InternalTenantContext context) throws UnableToObtainConnectionException, CallbackFailedException { return delegate.getMetrics(context); }
@VisibleForTesting void convertSamplesToScalarSamples(final String eventType, final Map<String, Object> inputSamples, final Map<Integer, ScalarSample> outputSamples, final InternalCallContext context) { if (inputSamples == null) { return; } final Integer eventCategoryId = timelineDAO.getOrAddEventCategory(eventType, context); for (final String attributeName : inputSamples.keySet()) { final Integer metricId = timelineDAO.getOrAddMetric(eventCategoryId, attributeName, context); final Object sample = inputSamples.get(attributeName); outputSamples.put(metricId, ScalarSample.fromObject(sample)); } }
final Integer hostId = dao.getOrAddSource(hostName, internalCallContext); Assert.assertNotNull(hostId); final Integer sampleOneId = dao.getOrAddMetric(eventCategoryId, sampleOne, internalCallContext); Assert.assertNotNull(sampleOneId); final String sampleTwo = UUID.randomUUID().toString(); final Integer sampleTwoId = dao.getOrAddMetric(eventCategoryId, sampleTwo, internalCallContext); Assert.assertNotNull(sampleTwoId); final BiMap<Integer, CategoryRecordIdAndMetric> sampleKinds = dao.getMetrics(internalCallContext); Assert.assertEquals(sampleKinds.size(), 2); Assert.assertEquals(sampleKinds.get(sampleOneId).getEventCategoryId(), (int) eventCategoryId); Assert.assertEquals(sampleKinds.get(sampleTwoId).getEventCategoryId(), (int) eventCategoryId); Assert.assertEquals(sampleKinds.get(sampleTwoId).getMetric(), sampleTwo); Assert.assertEquals(dao.getCategoryIdAndMetric(sampleOneId, internalCallContext).getEventCategoryId(), (int) eventCategoryId); Assert.assertEquals(dao.getCategoryIdAndMetric(sampleOneId, internalCallContext).getMetric(), sampleOne); Assert.assertEquals(dao.getCategoryIdAndMetric(sampleTwoId, internalCallContext).getEventCategoryId(), (int) eventCategoryId); Assert.assertEquals(dao.getCategoryIdAndMetric(sampleTwoId, internalCallContext).getMetric(), sampleTwo); dao.insertTimelineChunk(new TimelineChunk(0, hostId, sampleOneId, startTime, endTime, new byte[0], new byte[0], 0), internalCallContext); dao.insertTimelineChunk(new TimelineChunk(0, hostId, sampleTwoId, startTime, endTime, new byte[0], new byte[0], 0), internalCallContext); dao.insertTimelineChunk(new TimelineChunk(0, Integer.MAX_VALUE - 100, Integer.MAX_VALUE, startTime, endTime, new byte[0], new byte[0], 0), internalCallContext); dao.getSamplesBySourceIdsAndMetricIds(ImmutableList.<Integer>of(hostId), ImmutableList.<Integer>of(sampleOneId, sampleTwoId), startTime, startTime.plusSeconds(2), new TimelineChunkConsumer() { @Override public void processTimelineChunk(final TimelineChunk chunk) {
@Test(groups = "mysql") public void testAggregation() throws Exception { hostId = timelineDao.getOrAddSource(HOST_NAME, internalCallContext); Assert.assertNotNull(hostId); Assert.assertEquals(timelineDao.getSources(internalCallContext).values().size(), 1); minHeapUsedKindId = timelineDao.getOrAddMetric(EVENT_TYPE_ID, MIN_HEAPUSED_KIND, internalCallContext); Assert.assertNotNull(minHeapUsedKindId); maxHeapUsedKindId = timelineDao.getOrAddMetric(EVENT_TYPE_ID, MAX_HEAPUSED_KIND, internalCallContext); Assert.assertNotNull(maxHeapUsedKindId); Assert.assertEquals(timelineDao.getMetrics(internalCallContext).values().size(), 2); Assert.assertEquals(timelineDao.getSources(internalCallContext).values().size(), 1); Assert.assertEquals(timelineDao.getMetrics(internalCallContext).values().size(), 2);
@Override protected void writeJsonForChunks(final JsonGenerator generator, final Collection<? extends TimelineChunk> chunksForSourceAndMetric) throws IOException { for (final TimelineChunk chunk : chunksForSourceAndMetric) { final String source = timelineDao.getSource(chunk.getSourceId(), context); final CategoryRecordIdAndMetric categoryIdAndMetric = timelineDao.getCategoryIdAndMetric(chunk.getMetricId(), context); final String eventCategory = timelineDao.getEventCategory(categoryIdAndMetric.getEventCategoryId(), context); final String metric = categoryIdAndMetric.getMetric(); final String samples = CSVConsumer.getSamplesAsCSV(sampleCoder, chunk); // Don't write out empty samples if (!Strings.isNullOrEmpty(samples)) { generator.writeObject(new SamplesForMetricAndSource(source, eventCategory, metric, samples)); } } } }
@BeforeMethod(alwaysRun = true) public void setUp() throws Exception { Assert.assertTrue(basePath.mkdir()); System.setProperty("killbill.usage.timelines.spoolDir", basePath.getAbsolutePath()); final MeterConfig config = new ConfigurationObjectFactory(System.getProperties()).build(MeterConfig.class); timelineEventHandler = new TimelineEventHandler(config, dao, timelineCoder, sampleCoder, new BackgroundDBChunkWriter(dao, config, internalCallContextFactory), new FileBackedBuffer(config.getSpoolDir(), "TimelineEventHandler", 1024 * 1024, 10)); dao.getOrAddSource(HOST_UUID.toString(), internalCallContext); eventTypeId = dao.getOrAddEventCategory(EVENT_TYPE, internalCallContext); }
final Integer hostId = dao.getSourceId(HOST_UUID.toString(), internalCallContext); Assert.assertNotNull(hostId); final Integer sampleKindAId = dao.getMetricId(eventTypeId, SAMPLE_KIND_A, internalCallContext); Assert.assertNotNull(sampleKindAId); final Integer sampleKindBId = dao.getMetricId(eventTypeId, SAMPLE_KIND_B, internalCallContext); Assert.assertNotNull(sampleKindBId);
@Override public void getSamplesBySourceIdsAndMetricIds(final List<Integer> sourceIds, @Nullable final List<Integer> metricIds, final DateTime startTime, final DateTime endTime, final TimelineChunkConsumer chunkConsumer, final InternalTenantContext context) throws UnableToObtainConnectionException, CallbackFailedException { delegate.getSamplesBySourceIdsAndMetricIds(sourceIds, metricIds, startTime, endTime, chunkConsumer, context); }
@Override public int getOrAddSource(final String source, final InternalCallContext context) throws UnableToObtainConnectionException, CallbackFailedException { Integer sourceId = sourcesCache.inverse().get(source); if (sourceId == null) { sourceId = delegate.getOrAddSource(source, context); synchronized (sourcesCache) { if (sourcesCache.get(sourceId) == null) { sourcesCache.put(sourceId, source); } } } return sourceId; }
private List<Integer> translateSourcesToSourceIds(final List<String> sources) { final List<Integer> hostIds = new ArrayList<Integer>(sources.size()); for (final String source : sources) { final Integer sourceId = timelineDao.getSourceId(source, context); if (sourceId == null) { // Ignore continue; } hostIds.add(sourceId); } return hostIds; }
@Override public Long insertTimelineChunk(final TimelineChunk timelineChunk, final InternalCallContext context) throws UnableToObtainConnectionException, CallbackFailedException { return delegate.insertTimelineChunk(timelineChunk, context); }
@Override public BiMap<Integer, String> getSources(final InternalTenantContext context) throws UnableToObtainConnectionException, CallbackFailedException { return delegate.getSources(context); }
@Override public int getOrAddEventCategory(final String eventCategory, final InternalCallContext context) throws UnableToObtainConnectionException, CallbackFailedException { Integer eventCategoryId = eventCategoriesCache.inverse().get(eventCategory); if (eventCategoryId == null) { eventCategoryId = delegate.getOrAddEventCategory(eventCategory, context); synchronized (eventCategoriesCache) { if (eventCategoriesCache.get(eventCategoryId) == null) { eventCategoriesCache.put(eventCategoryId, eventCategory); } } } return eventCategoryId; }
@Override public int getOrAddMetric(final Integer eventCategoryId, final String metric, final InternalCallContext context) throws UnableToObtainConnectionException, CallbackFailedException { final CategoryRecordIdAndMetric categoryRecordIdAndMetric = new CategoryRecordIdAndMetric(eventCategoryId, metric); Integer metricId = metricsCache.inverse().get(categoryRecordIdAndMetric); if (metricId == null) { metricId = delegate.getOrAddMetric(eventCategoryId, metric, context); synchronized (metricsCache) { if (metricsCache.get(metricId) == null) { metricsCache.put(metricId, categoryRecordIdAndMetric); } } } return metricId; }
@Override public Integer getMetricId(final int eventCategoryId, final String metric, final InternalTenantContext context) throws UnableToObtainConnectionException { Integer result = metricsCache.inverse().get(new CategoryRecordIdAndMetric(eventCategoryId, metric)); if (result == null) { result = delegate.getMetricId(eventCategoryId, metric, context); if (result != null) { synchronized (metricsCache) { if (metricsCache.get(result) == null) { metricsCache.put(result, new CategoryRecordIdAndMetric(eventCategoryId, metric)); } } } } return result; }
@Override protected void writeJsonForChunks(final JsonGenerator generator, final Collection<? extends TimelineChunk> chunksForSourceAndMetric) throws IOException { for (final TimelineChunk chunk : chunksForSourceAndMetric) { final String source = timelineDao.getSource(chunk.getSourceId(), context); final CategoryRecordIdAndMetric categoryIdAndMetric = timelineDao.getCategoryIdAndMetric(chunk.getMetricId(), context); final String eventCategory = timelineDao.getEventCategory(categoryIdAndMetric.getEventCategoryId(), context); final String metric = categoryIdAndMetric.getMetric(); final String samples = CSVConsumer.getSamplesAsCSV(sampleCoder, chunk, accumulatorSampleConsumer); // Don't write out empty samples if (!Strings.isNullOrEmpty(samples)) { generator.writeObject(new SamplesForMetricAndSource(source, eventCategory, metric, samples)); } lastSource = source; lastEventCategory = eventCategory; lastMetric = metric; } }
System.setProperty("killbill.usage.timelines.spoolDir", basePath.getAbsolutePath()); final MeterConfig config = new ConfigurationObjectFactory(System.getProperties()).build(MeterConfig.class); final int eventTypeId = dao.getOrAddEventCategory(EVENT_TYPE, internalCallContext); final int int2shortId = dao.getOrAddMetric(eventTypeId, "int2short", internalCallContext); final int long2intId = dao.getOrAddMetric(eventTypeId, "long2int", internalCallContext); final int long2shortId = dao.getOrAddMetric(eventTypeId, "long2short", internalCallContext); final int int2intId = dao.getOrAddMetric(eventTypeId, "int2int", internalCallContext); final int long2longId = dao.getOrAddMetric(eventTypeId, "long2long", internalCallContext); final int hostId = 1; final TimelineEventHandler handler = new TimelineEventHandler(config, dao, timelineCoder, sampleCoder,
private void writeJsonForStoredChunks(final JsonGenerator generator, final List<Integer> hostIdsList, final List<Integer> sampleKindIdsList, final DateTime startTime, final DateTime endTime) throws IOException { final AtomicReference<Integer> lastHostId = new AtomicReference<Integer>(null); final AtomicReference<Integer> lastSampleKindId = new AtomicReference<Integer>(null); final List<TimelineChunk> chunksForHostAndSampleKind = new ArrayList<TimelineChunk>(); timelineDao.getSamplesBySourceIdsAndMetricIds(hostIdsList, sampleKindIdsList, startTime, endTime, new TimelineChunkConsumer() { @Override public void processTimelineChunk(final TimelineChunk chunks) { final Integer previousHostId = lastHostId.get(); final Integer previousSampleKindId = lastSampleKindId.get(); final Integer currentHostId = chunks.getSourceId(); final Integer currentSampleKindId = chunks.getMetricId(); chunksForHostAndSampleKind.add(chunks); if (previousHostId != null && (!previousHostId.equals(currentHostId) || !previousSampleKindId.equals(currentSampleKindId))) { try { writeJsonForChunks(generator, chunksForHostAndSampleKind); } catch (IOException e) { throw new RuntimeException(e); } chunksForHostAndSampleKind.clear(); } lastHostId.set(currentHostId); lastSampleKindId.set(currentSampleKindId); } }, context); if (chunksForHostAndSampleKind.size() > 0) { writeJsonForChunks(generator, chunksForHostAndSampleKind); chunksForHostAndSampleKind.clear(); } }