/** * Creates a new instance. */ public ConverterService() { converter = new MetricTimeSeriesConverter(); }
/** * Constructs a new Builder */ public Builder() { binaryTimeSeries = new BinaryTimeSeries(); }
/** * Sets the start of the time series * * @param start - the start time stamp * @return the builder */ public Builder start(long start) { binaryTimeSeries.setStart(start); return this; }
@Override public BinaryTimeSeries to(TimeSeries<Long, Double> timeSeries) { //-oo is represented through the first element that is null, hence if the size is one the time series is empty if (timeSeries.size() == 1) { LOGGER.info("Empty time series detected. {}", timeSeries); //Create a builder with the minimal required fields BinaryTimeSeries.Builder builder = new BinaryTimeSeries.Builder() .data(new byte[]{}) .start(0) .end(0); return builder.build(); } else { return new MetricTimeSeriesConverter().to(map(timeSeries)); } }
/** * Converts the solr document given in the constructor into a time series of type <T> * * @return a time series of type <T> * @throws Exception if bad things happen. */ @Override @SuppressWarnings("PMD.SignatureDeclareThrowsException") public T call() throws Exception { BinaryTimeSeries.Builder timeSeriesBuilder = new BinaryTimeSeries.Builder(); document.forEach(attributeField -> timeSeriesBuilder.field(attributeField.getKey(), attributeField.getValue())); LOGGER.debug("Calling document converter with {}", document); T timeSeries = documentConverter.from(timeSeriesBuilder.build(), queryStart, queryEnd); LOGGER.debug("Returning time series {} to callee", timeSeries); return timeSeries; } }
/** * Converts a solr document to a time series. * <p> * The resulting time series does not contain user defined attributes present in the solr document * (see {@link de.qaware.chronix.Schema#isUserDefined(String)} ). * * @param solrDoc the solr document * @return time series representing the given solr document */ public MetricTimeSeries toTimeSeries(SolrDocument solrDoc) { BinaryTimeSeries.Builder btsBuilder = new BinaryTimeSeries.Builder(); solrDoc.forEach(field -> btsBuilder.field(field.getKey(), field.getValue())); BinaryTimeSeries bts = btsBuilder.build(); long start = (long) solrDoc.get(START); long end = (long) solrDoc.get(END); return converter.from(bts, start, end); }
@Override public MetricTimeSeries from(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd) { LOGGER.debug("Converting {} to MetricTimeSeries starting at {} and ending at {}", binaryTimeSeries, queryStart, queryEnd); //get the metric String metric = binaryTimeSeries.get(MetricTSSchema.METRIC).toString(); //Third build a minimal time series MetricTimeSeries.Builder builder = new MetricTimeSeries.Builder(metric); //add all user defined attributes binaryTimeSeries.getFields().forEach((field, value) -> { if (MetricTSSchema.isUserDefined(field)) { builder.attribute(field, value); } }); //Default serialization is protocol buffers. if (binaryTimeSeries.getPoints().length > 0) { fromProtocolBuffers(binaryTimeSeries, queryStart, queryEnd, builder); } else if (binaryTimeSeries.getFields().containsKey(DATA_AS_JSON_FIELD)) { //do it from json fromJson(binaryTimeSeries, queryStart, queryEnd, builder); } else { //we have no data //set the start and end builder.start(binaryTimeSeries.getStart()); builder.end(binaryTimeSeries.getEnd()); } return builder.build(); }
private void fromProtocolBuffers(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd, MetricTimeSeries.Builder builder) { final InputStream decompressed = Compression.decompressToStream(binaryTimeSeries.getPoints()); ProtoBufMetricTimeSeriesSerializer.from(decompressed, binaryTimeSeries.getStart(), binaryTimeSeries.getEnd(), queryStart, queryEnd, builder); IOUtils.closeQuietly(decompressed); }
/** * Converts a time series of type <T> to SolInputDocument * * @param ts - the time series * @return a filled SolrInputDocument */ private static <T> SolrInputDocument convert(T ts, TimeSeriesConverter<T> converter) { BinaryTimeSeries series = converter.to(ts); SolrInputDocument solrDocument = new SolrInputDocument(); series.getFields().entrySet().forEach(entry -> solrDocument.addField(entry.getKey(), entry.getValue())); return solrDocument; } }
private void fromProtocolBuffers(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd, MetricTimeSeries.Builder builder) { final InputStream decompressed = Compression.decompressToStream(binaryTimeSeries.getPoints()); ProtoBufKassiopeiaSimpleSerializer.from(decompressed, binaryTimeSeries.getStart(), binaryTimeSeries.getEnd(), queryStart, queryEnd, builder); }
@Override public BinaryTimeSeries to(SimpleTimeSeries document) { BinaryTimeSeries.Builder binaryTimeSeries = new BinaryTimeSeries.Builder(); document.getFields().forEach(binaryTimeSeries::field); return binaryTimeSeries.build(); } }
/** * Converts a time series to a solr input document. * <p> * In order to avoid write conflicts, the resulting document does not contain * the attribute "_version_" and the attribute "id" is set to a random UUID. * * @param mts the time series * @return solr input document representing the given time series */ public SolrInputDocument toInputDocument(MetricTimeSeries mts) { SolrInputDocument inputDocument = new SolrInputDocument(); converter.to(mts) .getFields().entrySet() .stream() .filter(it -> !"_version_".equals(it.getKey())) .forEach(it -> inputDocument.addField(it.getKey(), it.getValue())); inputDocument.setField("id", UUID.randomUUID().toString()); return inputDocument; }
@Override public SimpleTimeSeries from(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd) { SimpleTimeSeries doc = new SimpleTimeSeries(); binaryTimeSeries.getFields().forEach(doc::add); return doc; }
@Override public TimeSeries<Long, Double> from(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd) { //This is a hack MetricTimeSeries metricTimeSeries = new MetricTimeSeriesConverter().from(binaryTimeSeries, queryStart, queryEnd); TimeSeries<Long, Double> timeSeries = new TimeSeries<>(map(metricTimeSeries.points())); metricTimeSeries.getAttributesReference().forEach(timeSeries::addAttribute); return timeSeries; }
private void fromJson(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd, MetricTimeSeries.Builder builder) { String jsonString = binaryTimeSeries.get(DATA_AS_JSON_FIELD).toString(); //Second deserialize JsonMetricTimeSeriesSerializer serializer = new JsonMetricTimeSeriesSerializer(); serializer.fromJson(jsonString.getBytes(Charset.forName(JsonMetricTimeSeriesSerializer.UTF_8)), queryStart, queryEnd, builder); }
private void fromJson(BinaryTimeSeries binaryTimeSeries, long queryStart, long queryEnd, MetricTimeSeries.Builder builder) { String jsonString = binaryTimeSeries.get(DATA_AS_JSON_FIELD).toString(); //Second deserialize JsonKassiopeiaSimpleSerializer serializer = new JsonKassiopeiaSimpleSerializer(); serializer.fromJson(jsonString.getBytes(Charset.forName(JsonKassiopeiaSimpleSerializer.UTF_8)), queryStart, queryEnd, builder); }
/** * Sets the end of the time series * * @param end - the end time stamp * @return the builder */ public Builder end(long end) { binaryTimeSeries.setEnd(end); return this; }
/** * Converts the solr document given in the constructor into a time series of type <T> * * @return a time series of type <T> * @throws Exception if bad things happen. */ @Override @SuppressWarnings("PMD.SignatureDeclareThrowsException") public T call() throws Exception { BinaryTimeSeries.Builder timeSeriesBuilder = new BinaryTimeSeries.Builder(); document.forEach(attributeField -> timeSeriesBuilder.field(attributeField.getKey(), attributeField.getValue())); LOGGER.debug("Calling document converter with {}", document); T timeSeries = documentConverter.from(timeSeriesBuilder.build(), queryStart, queryEnd); LOGGER.debug("Returning time series {} to callee", timeSeries); return timeSeries; } }
/** * Converts a time series of type <T> to SolInputDocument * * @param ts - the time series * @return a filled SolrInputDocument */ private static <T> SolrInputDocument convert(T ts, TimeSeriesConverter<T> converter) { BinaryTimeSeries series = converter.to(ts); SolrInputDocument solrDocument = new SolrInputDocument(); series.getFields().entrySet().forEach(entry -> solrDocument.addField(entry.getKey(), entry.getValue())); return solrDocument; } }
public ChronixUpdater(SolrClient solr, BlockingQueue<Record> records, Map<String, String> fieldToMetricTypeMapping, int batchSize, long flushInterval) { this.solr = solr; this.records = records; this.batchSize = batchSize; this.flushInterval = flushInterval; this.lastTS = System.nanoTime(); // far in the future ... converter = new MetricTimeSeriesConverter(); storage = new ChronixSolrStorage<>(batchSize, groupBy, reduce); if (fieldToMetricTypeMapping != null) { this.fieldToMetricTypeMapping.putAll(fieldToMetricTypeMapping); } //add the defaults this.fieldToMetricTypeMapping.put(FieldDictionary.RECORD_VALUE, RecordDictionary.METRIC); threadCount++; }