public void collect(BaseReportingKey baseReportingKey, long ts, double val) { aggregate(minute, new AggregationKey(baseReportingKey, ts / MINUTE), val); aggregate(hourly, new AggregationKey(baseReportingKey, ts / HOUR), val); aggregate(daily, new AggregationKey(baseReportingKey, ts / DAY), val); }
private static void aggregate(Map<AggregationKey, AggregationValue> map, AggregationKey key, double value) { AggregationValue aggregationValue = map.get(key); if (aggregationValue == null) { aggregationValue = map.putIfAbsent(key, new AggregationValue(value)); if (aggregationValue == null) { return; } } aggregationValue.update(value); }
@Override public void close() { System.out.println("Stopping aggregator..."); this.averageAggregator.close(); } }
for (AggregationKey keyToRemove : keys) { if (keyToRemove.isOutdated(nowTruncatedToPeriod)) { AggregationValue value = map.get(keyToRemove); FileUtils.getUserStorageDir(keyToRemove.getEmail(), keyToRemove.getAppName())); if (Files.notExists(userReportFolder)) { Files.createDirectories(userReportFolder); String fileName = generateFilename(keyToRemove.getDashId(), keyToRemove.getDeviceId(), keyToRemove.getPinType(), keyToRemove.getPin(), type); Path filePath = Paths.get(userReportFolder.toString(), fileName); FileUtils.write(filePath, value.calcAverage(), keyToRemove.getTs(type));
public static void prepareReportingInsert(PreparedStatement ps, String email, int dashId, int deviceId, short pin, PinType pinType, long ts, double value) throws SQLException { ps.setString(1, email); ps.setInt(2, dashId); ps.setInt(3, deviceId); ps.setShort(4, pin); ps.setInt(5, pinType.ordinal()); ps.setTimestamp(6, new Timestamp(ts), DateTimeUtils.UTC_CALENDAR); ps.setDouble(7, value); }
final Object value = entry.getValue(); ps.setString(1, key.getEmail()); ps.setInt(2, key.getDashId()); ps.setInt(3, key.getDeviceId()); ps.setShort(4, key.getPin()); ps.setString(5, key.getPinType().pinTypeString); ps.setTimestamp(6, new Timestamp(key.ts), DateTimeUtils.UTC_CALENDAR);
@Override public void run() { try { Map<AggregationKey, AggregationValue> removedKeysMinute = process(reportingDao.averageAggregator.getMinute(), GraphGranularityType.MINUTE); Map<AggregationKey, AggregationValue> removedKeysHour = process(reportingDao.averageAggregator.getHourly(), GraphGranularityType.HOURLY); Map<AggregationKey, AggregationValue> removedKeysDay = process(reportingDao.averageAggregator.getDaily(), GraphGranularityType.DAILY); reportingDBManager.insertReporting(removedKeysMinute, GraphGranularityType.MINUTE); reportingDBManager.insertReporting(removedKeysHour, GraphGranularityType.HOURLY); reportingDBManager.insertReporting(removedKeysDay, GraphGranularityType.DAILY); reportingDBManager.insertReportingRaw(reportingDao.rawDataProcessor.rawStorage); reportingDBManager.cleanOldReportingRecords(Instant.now()); } catch (Exception e) { log.error("Error during reporting job.", e); } }
public void collect(BaseReportingKey key, long ts, String stringValue, double doubleValue) { final AggregationKey aggregationKey = new AggregationKey(key, ts); if (doubleValue == NumberUtil.NO_RESULT) { rawStorage.put(aggregationKey, stringValue); } else { rawStorage.put(aggregationKey, doubleValue); } }
public ReportingDiskDao(String reportingFolder, boolean isEnabled) { this.averageAggregator = new AverageAggregatorProcessor(reportingFolder); this.rawDataCacheForGraphProcessor = new RawDataCacheForGraphProcessor(); this.dataFolder = reportingFolder; this.enableRawDbDataStore = isEnabled; this.rawDataProcessor = new RawDataProcessor(enableRawDbDataStore); this.csvGenerator = new CSVGenerator(this); createCSVFolder(); }
private void process(User user, DashBoard dash, int deviceId, short pin, PinType pinType, String value, long ts, double doubleVal) { if (enableRawDbDataStore) { rawDataProcessor.collect( new BaseReportingKey(user.email, user.appName, dash.id, deviceId, pinType, pin), ts, value, doubleVal); } //not a number, nothing to aggregate if (doubleVal == NumberUtil.NO_RESULT) { return; } //store history data only for the pins assigned to the superchart Widget widgetWithLogPins = user.profile.getWidgetWithLoggedPin(dash, deviceId, pin, pinType); if (widgetWithLogPins != null) { BaseReportingKey key = new BaseReportingKey(user.email, user.appName, dash.id, deviceId, pinType, pin); averageAggregator.collect(key, ts, doubleVal); if (widgetWithLogPins instanceof Superchart) { if (((Superchart) widgetWithLogPins).hasLivePeriodsSelected()) { rawDataCacheForGraphProcessor.collect(key, new GraphValue(doubleVal, ts)); } } } }