@Override public Iterator<Entry> createIterator(final Split split) { InputSplit s = (InputSplit) split; return read(s.key, s.startTime, s.endTime, s.tags); }
/** * Writes an entry to the Dataset. This method overrides {@code write(key, value)} in {@link BatchWritable}. * The key is ignored in this method and instead it uses the key provided in the <code>Entry</code> object. * * @param key row key to write to. Value is ignored * @param value entry to write. The key used to write to the table is extracted from this object */ @WriteOnly @Override public void write(final byte[] key, final Entry value) { write(value); }
@Override public TimeseriesTable getDataset(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> arguments, ClassLoader classLoader) throws IOException { return new TimeseriesTable(spec, this.<Table>getDataset(datasetContext, "ts", spec, arguments, classLoader)); }
@Test public void testColumnNameFormatWithNoTags() { long ts = System.currentTimeMillis(); byte[] columnName = TimeseriesTable.createColumnName(ts, new byte[0][]); Assert.assertEquals(ts, TimeseriesTable.parseTimeStamp(columnName)); Assert.assertFalse(TimeseriesTable.hasTags(columnName)); }
table.write(m1e1); TimeseriesTable.Entry m1e2 = new TimeseriesTable.Entry(metric1, Bytes.toBytes(10L), ts + 2 * second, tag3); table.write(m1e2); TimeseriesTable.Entry m1e3 = new TimeseriesTable.Entry(metric1, Bytes.toBytes(15L), ts + 2 * hour, tag1); table.write(m1e3); TimeseriesTable.Entry m1e4 = new TimeseriesTable.Entry(metric1, Bytes.toBytes(23L), ts + 3 * hour, tag2, tag3); table.write(m1e4); TimeseriesTable.Entry m1e5 = new TimeseriesTable.Entry(metric1, Bytes.toBytes(55L), ts + 3 * hour + 2 * second); table.write(m1e5); table.write(m2e1); TimeseriesTable.Entry m2e2 = new TimeseriesTable.Entry(metric2, Bytes.toBytes(11L), ts + 2 * second, tag2); table.write(m2e2); TimeseriesTable.Entry m2e3 = new TimeseriesTable.Entry(metric2, Bytes.toBytes(16L), ts + 2 * hour, tag2); table.write(m2e3); TimeseriesTable.Entry m2e4 = new TimeseriesTable.Entry(metric2, Bytes.toBytes(24L), ts + 3 * hour, tag1, tag3); table.write(m2e4); TimeseriesTable.Entry m2e5 = new TimeseriesTable.Entry(metric2, Bytes.toBytes(56L), ts + 3 * hour + 2 * second, tag3, tag1); table.write(m2e5); assertReadResult(table.read(metric1, ts, ts + 5 * hour), m1e1, m1e2, m1e3, m1e4, m1e5); assertReadResult(table.read(metric1, ts, ts + 5 * hour, tag2), m1e1, m1e4); assertReadResult(table.read(metric1, ts, ts + 5 * hour, tag4)); assertReadResult(table.read(metric1, ts, ts + 5 * hour, tag2, tag4)); assertReadResult(table.read(metric1, 0, Long.MAX_VALUE)); assertReadResult(table.read(metric1, ts, ts + 5 * hour, 1, 2), m1e2, m1e3);
Assert.assertArrayEquals(TimeseriesTable.createRow(key, ts1, timeIntervalPerRow), TimeseriesTable.createRow(key, ts1 + timeIntervalPerRow / 2, timeIntervalPerRow)); Assert.assertFalse(Arrays.equals(TimeseriesTable.createRow(key, ts1, timeIntervalPerRow), TimeseriesTable.createRow(key, ts1 + timeIntervalPerRow * 2, timeIntervalPerRow))); Bytes.compareTo(TimeseriesTable.createRow(key, ts1, timeIntervalPerRow), TimeseriesTable.createRow(key, ts1 + timeIntervalPerRow * 5, timeIntervalPerRow)) < 0); Assert.assertFalse(Arrays.equals(TimeseriesTable.createRow(key, ts1, timeIntervalPerRow), TimeseriesTable.createRow(key2, ts1, timeIntervalPerRow))); long endTime = timeIntervalPerRow * 100; for (long i = startTime; i <= endTime; i += timeIntervalPerRow / 10) { byte[] row = TimeseriesTable.createRow(key, i, timeIntervalPerRow); if (rows.isEmpty() || !Arrays.equals(rows.get(rows.size() - 1), row)) { rows.add(row); int intervalsCount = (int) TimeseriesTable.getTimeIntervalsCount(startTime, endTime, timeIntervalPerRow); byte[][] rowsForInterval = new byte[intervalsCount][]; for (int i = 0; i < intervalsCount; i++) { rowsForInterval[i] = TimeseriesTable.getRowOfKthInterval(key, startTime, i, timeIntervalPerRow); Assert.assertEquals(2, TimeseriesTable.getTimeIntervalsCount(timeIntervalPerRow + 1, timeIntervalPerRow * 2 + 1, timeIntervalPerRow));
@Override public void initialize() throws Exception { MapReduceContext context = getContext(); metrics.count("beforeSubmit", 1); Job hadoopJob = context.getHadoopJob(); AggregateMetricsByTag.configureJob(hadoopJob); String metricName = context.getRuntimeArguments().get("metric"); Long startTs = Long.valueOf(context.getRuntimeArguments().get("startTs")); Long stopTs = Long.valueOf(context.getRuntimeArguments().get("stopTs")); String tag = context.getRuntimeArguments().get("tag"); context.addInput(Input.ofDataset("timeSeries", table.getInputSplits(2, Bytes.toBytes(metricName), startTs, stopTs, Bytes.toBytes(tag)))); beforeSubmitTable.write(Bytes.toBytes("beforeSubmit"), Bytes.toBytes("beforeSubmit:done")); String frequentFlushing = context.getRuntimeArguments().get("frequentFlushing"); if (frequentFlushing != null) { hadoopJob.getConfiguration().setInt("c.mapper.flush.freq", 1); hadoopJob.getConfiguration().setInt("c.reducer.flush.freq", 1); } metrics.count("beforeSubmit", 1); context.addOutput(Output.ofDataset("timeSeries")); }
/** * Time series table format test. */ @Test public void testColumnNameFormat() { // Note: tags are sorted lexographically byte[] tag1 = Bytes.toBytes("111"); byte[] tag2 = Bytes.toBytes("22"); byte[] tag3 = Bytes.toBytes("3"); long ts = System.currentTimeMillis(); byte[][] tags = { tag1, tag2, tag3 }; byte[] columnName = TimeseriesTable.createColumnName(ts, tags); Assert.assertEquals(ts, TimeseriesTable.parseTimeStamp(columnName)); Assert.assertTrue(TimeseriesTable.hasTags(columnName)); }
public final Iterator<Entry> read(byte[] key, long startTime, long endTime, int offset, final int limit, byte[]... tags) { final Iterator<Entry> iterator = read(key, startTime, endTime, tags); int advance = offset; while (advance > 0 && iterator.hasNext()) {
/** * Writes an entry to the Dataset. * * @param entry entry to write */ @WriteOnly public final void write(Entry entry) { write(entry.getKey(), entry.getValue(), entry.getTimestamp(), entry.getTags()); }
/** * Tests no entry returned in the time interval. */ private void testNoRow(long now) { long endTs = now + TimeUnit.MINUTES.toMillis(20); long startTs = now + TimeUnit.MINUTES.toMillis(10); Iterator<TimeseriesTable.Entry> entryIterator = table.read(ALL_KEY, startTs, endTs); Assert.assertFalse("No entry returned in the time interval", entryIterator.hasNext()); }
@Override public void apply() throws Exception { long ts = fact.getTs(); byte[] srcTag = Bytes.toBytes(fact.getDimensions().get(SRC_DEVICE_ID_TAG)); byte[] destTag = Bytes.toBytes(fact.getDimensions().get(DEST_DEVICE_ID_TAG)); if (fact.getDimensions().get(SRC_DEVICE_ID_TAG).equals("1.1.1.1")) { table.write(new TimeseriesTable.Entry(ALL_KEY, Bytes.toBytes(fact.getDimensions().get(DST_TAG)), ts)); } else { table.write(new TimeseriesTable.Entry(ALL_KEY, Bytes.toBytes(fact.getDimensions().get(DST_TAG)), ts, srcTag, destTag)); } } });
@Override public void apply() throws Exception { long ts = System.currentTimeMillis(); table.read(Bytes.toBytes("any"), ts, ts - 100); } });
private void fillTestInputData(TimeseriesTable table, boolean withBadData) { byte[] metric1 = Bytes.toBytes("metric"); byte[] metric2 = Bytes.toBytes("metric2"); byte[] tag1 = Bytes.toBytes("tag1"); byte[] tag2 = Bytes.toBytes("tag2"); byte[] tag3 = Bytes.toBytes("tag3"); // m1e1 = metric: 1, entity: 1 table.write(new TimeseriesTable.Entry(metric1, Bytes.toBytes(3L), 1, tag3, tag2, tag1)); table.write(new TimeseriesTable.Entry(metric1, Bytes.toBytes(10L), 2, tag2, tag3)); // 55L will make job fail table.write(new TimeseriesTable.Entry(metric1, Bytes.toBytes(withBadData ? 55L : 15L), 3, tag1, tag3)); table.write(new TimeseriesTable.Entry(metric1, Bytes.toBytes(23L), 4, tag2)); table.write(new TimeseriesTable.Entry(metric2, Bytes.toBytes(4L), 3, tag1, tag3)); }
/** * Tests reading entries without tag. */ private void testReadEntryWithoutTag(long now) { long endTs = now; long startTs = now - TimeUnit.SECONDS.toMillis(30); List<TimeseriesTable.Entry> result = Lists.newArrayList(); // f6 and f7 are read back, since they are written to the table without tag. Iterator<TimeseriesTable.Entry> entryIterator = table.read(ALL_KEY, startTs, endTs); while (entryIterator.hasNext()) { result.add(entryIterator.next()); } Assert.assertEquals(2, result.size()); }
/** * Tests iterator filter. queryTags will be sorted inside EntryScanner. */ private void testFilter(long now) { long endTs = now; long startTs = now - TimeUnit.MINUTES.toMicros(37); String[] queryTags = {"device2", "device1"}; Iterator<TimeseriesTable.Entry> entryIterator = table.read(ALL_KEY, startTs, endTs, Bytes.toByteArrays(queryTags)); List<TimeseriesTable.Entry> result = Lists.newArrayList(); while (entryIterator.hasNext()) { TimeseriesTable.Entry entry = entryIterator.next(); result.add(entry); } Assert.assertEquals(2, result.size()); }
@Override public void apply() throws Exception { long ts = System.currentTimeMillis(); Iterator<TimeseriesTable.Entry> temp = table.read(Bytes.toBytes("any"), ts, ts); Assert.assertFalse(temp.hasNext()); temp = table.read(Bytes.toBytes("any"), ts, ts + 100); Assert.assertFalse(temp.hasNext()); } });
/** * Tests queryTags are not matched. */ private void testNoTagMatch(long now) { long endTs = now; long startTs = now - TimeUnit.MINUTES.toMicros(37); String[] queryTags = {"device5", "device2"}; Iterator<TimeseriesTable.Entry> entryIterator = table.read(ALL_KEY, startTs, endTs, Bytes.toByteArrays(queryTags)); Assert.assertFalse(entryIterator.hasNext()); }
private long getCounts(String word, TimeseriesTable tsTable) { long result = 0; Iterator<TimeseriesTable.Entry> itor = tsTable.read(Bytes.toBytes(word), 0, Long.MAX_VALUE); while (itor.hasNext()) { result += Bytes.toLong(itor.next().getValue()); } return result; } }
/** * Tests the correctness of facts fetched by EntryScanner. */ private void testScan(long now) { long endTs = now; long startTs = now - TimeUnit.MINUTES.toMillis(18) + 1; // read facts in time window (now - 18mins, now) Iterator<TimeseriesTable.Entry> entryIterator = table.read(ALL_KEY, startTs, endTs); // dsts of the facts sent from the earliest ts to now ImmutableSet<String> expectedDsts = ImmutableSet.of("20.123.8.60", "20.123.8.50", "10.123.8.40", "10.123.8.30", "10.123.8.20"); Set<String> actualDsts = Sets.newHashSet(); while (entryIterator.hasNext()) { TimeseriesTable.Entry entry = entryIterator.next(); actualDsts.add(Bytes.toString(entry.getValue())); } Assert.assertEquals(5, expectedDsts.size()); Assert.assertEquals("dst is not correct", expectedDsts, actualDsts); }