@Override public void apply() { delegate.delete(row, columns); } });
@Nullable @Override public byte[] get(byte[] row, byte[] column) { return v3HBaseTable.get(row, column); }
@Override public void increment(NavigableMap<byte[], NavigableMap<byte[], Long>> updates) { v3HBaseTable.increment(updates); }
try (Scanner scanner = v2MetricsTable.scan(null, null, null)) { LOG.info("Starting Metrics Data Migration from {} to {} with {}ms sleep between records", v2MetricsTableName, v3MetricsTableName, sleepMillisBetweenRecords); byte[] value = v3MetricsTable.get(rowKey, entry.getKey()); if (value == null) { if (v3MetricsTable.swap(rowKey, entry.getKey(), null, entry.getValue())) { gaugeDeletes.add(entry.getKey()); } else { v2MetricsTable.delete(rowKey, deletes); v3MetricsTable.increment(rowIncrements);
@Test public void testDeleteIncrements() throws Exception { // note: this is pretty important test case for tables with counters, e.g. metrics MetricsTable table = getTable("testDeleteIncrements"); // delete increment and increment again table.increment(A, ImmutableMap.of(B, 5L)); table.increment(A, ImmutableMap.of(B, 2L)); Assert.assertEquals(7L, Bytes.toLong(table.get(A, B))); table.delete(A, new byte[][]{B}); Assert.assertNull(table.get(A, B)); table.increment(A, ImmutableMap.of(B, 3L)); Assert.assertEquals(3L, Bytes.toLong(table.get(A, B))); }
byte[] rowKey = Bytes.toBytes(key.getType() + '.' + key.getName()); byte[] result = table.get(rowKey, ID); long newId = table.incrementAndGet(maxIdRowKey, MAX_ID, 1L); table.swap(maxIdRowKey, MAX_ID, Bytes.toBytes(maxId), Bytes.toBytes(newId)); if (table.swap(rowKey, ID, null, Bytes.toBytes(newId))) { while (!table.swap(rowKey, NAME, oldName, Bytes.toBytes(key.getName()))) { result = table.get(rowKey, NAME); if (result == null) { throw new IllegalStateException("Fail to set reverse mapping from id to name."); result = table.get(rowKey, ID);
@Test public void testDelete() throws Exception { MetricsTable table = getTable("testDelete"); NavigableMap<byte[], SortedMap<byte[], Long>> writes = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (int i = 0; i < 1024; i++) { writes.put(Bytes.toBytes(i << 22), mapOf(A, Bytes.toLong(X))); } table.put(writes); // verify the first and last are there (sanity test for correctness of test logic) Assert.assertArrayEquals(X, table.get(Bytes.toBytes(0x00000000), A)); Assert.assertArrayEquals(X, table.get(Bytes.toBytes(0xffc00000), A)); List<byte[]> toDelete = ImmutableList.of( Bytes.toBytes(0xa1000000), Bytes.toBytes(0xb2000000), Bytes.toBytes(0xc3000000)); // verify these three are there, and delete them for (byte[] row : toDelete) { Assert.assertArrayEquals(X, table.get(row, A)); table.delete(row, new byte[][] {A}); } // verify these three are now gone. for (byte[] row : toDelete) { Assert.assertNull(table.get(row, A)); } // verify nothing else is gone by counting all entries in a scan Assert.assertEquals(1021, countRange(table, null, null)); }
table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR) .put(A, mapOf(P, Bytes.toLong(X), Q, Bytes.toLong(Y))) .put(B, mapOf(P, Bytes.toLong(X), R, Bytes.toLong(Z))).build()); Assert.assertEquals(Bytes.toLong(X), Bytes.toLong(table.get(A, P))); Assert.assertEquals(Bytes.toLong(Y), Bytes.toLong(table.get(A, Q))); Assert.assertNull(table.get(A, R)); Assert.assertEquals(Bytes.toLong(X), Bytes.toLong(table.get(B, P))); Assert.assertEquals(Bytes.toLong(Z), Bytes.toLong(table.get(B, R))); Assert.assertNull(table.get(B, Q)); table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR) .put(A, mapOf(P, Bytes.toLong(A), R, Bytes.toLong(C))).build()); Assert.assertEquals(Bytes.toLong(A), Bytes.toLong(table.get(A, P))); Assert.assertEquals(Bytes.toLong(Y), Bytes.toLong(table.get(A, Q))); Assert.assertEquals(Bytes.toLong(C), Bytes.toLong(table.get(A, R))); Assert.assertTrue(table.swap(A, P, A, B)); Assert.assertEquals(Bytes.toLong(B), Bytes.toLong(table.get(A, P))); Assert.assertFalse(table.swap(A, P, A, B)); Assert.assertArrayEquals(B, table.get(A, P)); Assert.assertFalse(table.swap(B, Q, A, B)); Assert.assertNull(table.get(B, Q)); Assert.assertTrue(table.swap(A, P, B, null)); Assert.assertNull(table.get(A, P)); Assert.assertFalse(table.swap(A, Q, A, null));
@Override public Scanner call() { return delegate.scan(start, stop, filter); } });
public void add(List<Fact> facts) { // Simply collecting all rows/cols/values that need to be put to the underlying table. NavigableMap<byte[], NavigableMap<byte[], byte[]>> gaugesTable = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); NavigableMap<byte[], NavigableMap<byte[], byte[]>> incrementsTable = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Fact fact : facts) { for (Measurement measurement : fact.getMeasurements()) { byte[] rowKey = codec.createRowKey(fact.getDimensionValues(), measurement.getName(), fact.getTimestamp()); byte[] column = codec.createColumn(fact.getTimestamp()); if (MeasureType.COUNTER == measurement.getType()) { inc(incrementsTable, rowKey, column, measurement.getValue()); } else { set(gaugesTable, rowKey, column, Bytes.toBytes(measurement.getValue())); } } } NavigableMap<byte[], NavigableMap<byte[], Long>> convertedIncrementsTable = Maps.transformValues(incrementsTable, TRANSFORM_MAP_BYTE_ARRAY_TO_LONG); NavigableMap<byte[], NavigableMap<byte[], Long>> convertedGaugesTable = Maps.transformValues(gaugesTable, TRANSFORM_MAP_BYTE_ARRAY_TO_LONG); // todo: replace with single call, to be able to optimize rpcs in underlying table timeSeriesTable.put(convertedGaugesTable); timeSeriesTable.increment(convertedIncrementsTable); if (metrics != null) { metrics.increment(putCountMetric, convertedGaugesTable.size()); metrics.increment(incrementCountMetric, convertedIncrementsTable.size()); } }
@Test public void testConcurrentSwap() throws Exception { MetricsTable table = getTable("testConcurrentSwap"); final long rounds = 500; table.put(ImmutableSortedMap.<byte[], SortedMap<byte[], Long>>orderedBy(Bytes.BYTES_COMPARATOR) .put(A, mapOf(B, 0L)).build()); AtomicInteger[] counts = { new AtomicInteger(), new AtomicInteger() }; // [0] for success, [1] for failures Thread t1 = new SwapThread(table, A, B, counts, rounds); Thread t2 = new SwapThread(table, A, B, counts, rounds); t1.start(); t2.start(); t1.join(); t2.join(); Assert.assertEquals(rounds, Bytes.toLong(table.get(A, B))); Assert.assertEquals(rounds, counts[0].get()); // number of successful swaps Assert.assertEquals(rounds, counts[1].get()); // number of failed swaps }
@Override public void close() throws IOException { table.close(); }
@Override public void apply() { delegate.put(updates); } });
@Override public void putBytes(SortedMap<byte[], ? extends SortedMap<byte[], byte[]>> updates) { v3HBaseTable.putBytes(updates); }
@Override public boolean swap(byte[] row, byte[] column, byte[] oldValue, byte[] newValue) { return v3HBaseTable.swap(row, column, oldValue, newValue); }
@Override public long incrementAndGet(byte[] row, byte[] column, long delta) { // This method will not be called from FactTable return v3HBaseTable.incrementAndGet(row, column, delta); }
byte[] rowKey = Bytes.toBytes(key.getType() + '.' + key.getName()); byte[] result = table.get(rowKey, ID); long newId = table.incrementAndGet(maxIdRowKey, MAX_ID, 1L); table.swap(maxIdRowKey, MAX_ID, Bytes.toBytes(maxId), Bytes.toBytes(newId)); if (table.swap(rowKey, ID, null, Bytes.toBytes(newId))) { while (!table.swap(rowKey, NAME, oldName, Bytes.toBytes(key.getName()))) { result = table.get(rowKey, NAME); if (result == null) { throw new IllegalStateException("Fail to set reverse mapping from id to name."); result = table.get(rowKey, ID);
@Override public Scanner scan(@Nullable byte[] start, @Nullable byte[] stop, @Nullable FuzzyRowFilter filter) { Scanner v2Scan = null; if (v2HBaseTable != null) { try { v2Scan = v2HBaseTable.scan(start, stop, filter); } catch (Exception e) { handleV2TableException(e, "Scan", getV2MetricsTableDatasetId().getDataset()); } } Scanner v3Scan = v3HBaseTable.scan(start, stop, filter); return new CombinedMetricsScanner(v2Scan, v3Scan, getV2MetricsTableDatasetId(), datasetFramework); }