@Override public void deleteRange(TableReference tableRef, RangeRequest range) { try (ClosableIterator<RowResult<Set<Long>>> iterator = getRangeOfTimestamps(tableRef, range, AtlasDbConstants.MAX_TS)) { while (iterator.hasNext()) { RowResult<Set<Long>> rowResult = iterator.next(); Multimap<Cell, Long> cellsToDelete = HashMultimap.create(); rowResult.getCells().forEach(entry -> cellsToDelete.putAll(entry.getKey(), entry.getValue())); delete(tableRef, cellsToDelete); } } }
@Override public void deleteRange(TableReference tableRef, RangeRequest range) { try (ClosableIterator<RowResult<Set<Long>>> iterator = getRangeOfTimestamps(tableRef, range, AtlasDbConstants.MAX_TS)) { while (iterator.hasNext()) { RowResult<Set<Long>> rowResult = iterator.next(); Multimap<Cell, Long> cellsToDelete = HashMultimap.create(); for (Entry<Cell, Set<Long>> entry : rowResult.getCells()) { cellsToDelete.putAll(entry.getKey(), entry.getValue()); } delete(tableRef, cellsToDelete); } } }
public static void run(KeyValueService kvs, PrintWriter output, int batchSize) { Context context = new Context(kvs, output, batchSize); // This potentially needs to iterate multiple times because getRange // only grabs the most recent timestamp for each cell, but we care // about every timestamp in the old scrub queue (this was the main // problem with the old scrub queue). Each iteration will peel off the // top version of each cell until the entire queue is drained. for (int i = 0;; i++) { output.println("Starting iteration " + i + " of scrub migration."); Stopwatch watch = Stopwatch.createStarted(); try (ClosableIterator<RowResult<Value>> iter = kvs.getRange( AtlasDbConstants.OLD_SCRUB_TABLE, RangeRequest.all(), Long.MAX_VALUE)) { if (!iter.hasNext()) { output.println("Finished all iterations of scrub migration."); break; } runOnce(context, iter); } output.println("Finished iteration " + i + " of scrub migration in " + watch.elapsed(TimeUnit.SECONDS) + " seconds."); } }
void verifyTableSwept(TableReference tableRef, int expectedCells, boolean conservative) { try (ClosableIterator<RowResult<Set<Long>>> iter = kvs.getRangeOfTimestamps(tableRef, RangeRequest.all(), Long.MAX_VALUE)) { int numCells = 0; while (iter.hasNext()) { RowResult<Set<Long>> rr = iter.next(); numCells += rr.getColumns().size(); Assert.assertTrue(String.format("Found unswept values in %s!", tableRef.getQualifiedName()), rr.getColumns().values().stream() .allMatch(s -> s.size() == 1 || (conservative && s.size() == 2 && s.contains(-1L)))); } Assert.assertEquals(expectedCells, numCells); } }
@Test(expected = RuntimeException.class) public void testGetRangeOfTimestampsThrowsOnError() { keyValueService.getRangeOfTimestamps(TEST_NONEXISTING_TABLE, RangeRequest.all(), AtlasDbConstants.MAX_TS) .hasNext(); }
private static Map<Cell, Long> doGetLatestTimestamps(DbReadTable table, Map<Cell, Long> timestampByCell) { try (ClosableIterator<AgnosticLightResultRow> iter = table.getLatestCells(timestampByCell, false)) { Map<Cell, Long> results = Maps.newHashMap(); while (iter.hasNext()) { AgnosticLightResultRow row = iter.next(); Cell cell = Cell.create(row.getBytes(ROW), row.getBytes(COL)); long ts = row.getLong(TIMESTAMP); Long oldTs = results.put(cell, ts); if (oldTs != null && oldTs > ts) { results.put(cell, oldTs); } } return results; } }
@Override public Map<Long, byte[]> loadOverflowValues(ConnectionSupplier conns, TableReference tableRef, Collection<Long> overflowIds) { if (overflowIds.isEmpty()) { return Collections.emptyMap(); } else { Map<Long, byte[]> ret = Maps.newHashMapWithExpectedSize(overflowIds.size()); for (FullQuery query : getOverflowQueries(conns, tableRef, overflowIds)) { try (ClosableIterator<AgnosticLightResultRow> overflowIter = select(conns, query)) { while (overflowIter.hasNext()) { AgnosticLightResultRow row = overflowIter.next(); // QA-94468 LONG RAW typed columns ("val" in this case) must be retrieved first from the result // set. See https://docs.oracle.com/cd/B19306_01/java.102/b14355/jstreams.htm#i1007581 byte[] val = row.getBytes("val"); long id = row.getLong("id"); ret.put(id, val); } } } return ret; } }
private static Multimap<Cell, Long> doGetAllTimestamps(DbReadTable table, Iterable<Cell> cells, long timestamp) { try (ClosableIterator<AgnosticLightResultRow> iter = table.getAllCells(cells, timestamp, false)) { Multimap<Cell, Long> results = ArrayListMultimap.create(); while (iter.hasNext()) { AgnosticLightResultRow row = iter.next(); Cell cell = Cell.create(row.getBytes(ROW), row.getBytes(COL)); long ts = row.getLong(TIMESTAMP); results.put(cell, ts); } return results; } }
@Test public void testGetRangeThrowsOnError() { try { keyValueService.getRange(TEST_NONEXISTING_TABLE, RangeRequest.all(), AtlasDbConstants.MAX_TS).hasNext(); Assert.fail("getRange must throw on failure"); } catch (RuntimeException e) { // Expected } }
private Map<Sha256Hash, Integer> getColumnCountsUnordered(TableReference tableRef, List<byte[]> rowList, ColumnRangeSelection columnRangeSelection, long timestamp) { return runRead(tableRef, dbReadTable -> { Map<Sha256Hash, Integer> counts = new HashMap<>(rowList.size()); try (ClosableIterator<AgnosticLightResultRow> iter = dbReadTable.getRowsColumnRangeCounts(rowList, timestamp, columnRangeSelection)) { while (iter.hasNext()) { AgnosticLightResultRow row = iter.next(); Sha256Hash rowHash = Sha256Hash.computeHash(row.getBytes(ROW)); counts.put(rowHash, row.getInteger("column_count")); } } return counts; }); }
@Test public void testDeleteMultipleVersions() { putTestDataForMultipleTimestamps(); ClosableIterator<RowResult<Value>> result = keyValueService.getRange( TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 1); assertTrue(result.hasNext()); keyValueService.delete(TEST_TABLE, ImmutableMultimap.of(TEST_CELL, TEST_TIMESTAMP)); result = keyValueService.getRange(TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 1); assertFalse(result.hasNext()); result = keyValueService.getRange(TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 2); assertTrue(result.hasNext()); }
private void verifyCheckAndSet(Cell key, byte[] expectedValue) { Multimap<Cell, Long> timestamps = keyValueService.getAllTimestamps(TEST_TABLE, ImmutableSet.of(key), 1L); assertEquals(1, timestamps.size()); assertTrue(timestamps.containsEntry(key, AtlasDbConstants.TRANSACTION_TS)); ClosableIterator<RowResult<Value>> result = keyValueService.getRange(TEST_TABLE, RangeRequest.all(), AtlasDbConstants.TRANSACTION_TS + 1); // Check result is right byte[] actual = result.next().getColumns().get(key.getColumnName()).getContents(); assertArrayEquals(String.format("Value \"%s\" different from expected \"%s\"", new String(actual, StandardCharsets.UTF_8), new String(expectedValue, StandardCharsets.UTF_8)), expectedValue, actual); // Check no more results assertFalse(result.hasNext()); }
private static void runOnce(Context context, ClosableIterator<RowResult<Value>> iter) { Multimap<Cell, Value> batchToCreate = ArrayListMultimap.create(); Multimap<Cell, Long> batchToDelete = ArrayListMultimap.create(); while (iter.hasNext()) { RowResult<Value> rowResult = iter.next(); byte[] row = rowResult.getRowName(); for (Entry<byte[], Value> entry : rowResult.getColumns().entrySet()) { byte[] col = entry.getKey(); Value value = entry.getValue(); long timestamp = value.getTimestamp(); String[] tableNames = StringUtils.split( PtBytes.toString(value.getContents()), AtlasDbConstants.OLD_SCRUB_TABLE_SEPARATOR_CHAR); for (String tableName : tableNames) { byte[] tableBytes = EncodingUtils.encodeVarString(tableName); byte[] newCol = EncodingUtils.add(tableBytes, col); Cell newCell = Cell.create(row, newCol); batchToCreate.put(newCell, Value.create(DUMMY_CONTENTS, timestamp)); } batchToDelete.put(Cell.create(row, col), timestamp); if (batchToDelete.size() >= context.batchSize) { flush(context, batchToCreate, batchToDelete); } } } if (!batchToDelete.isEmpty()) { flush(context, batchToCreate, batchToDelete); } }
private void testGetRangeWithTimestamps(boolean reverse) { putTestDataForMultipleTimestamps(); final RangeRequest range; if (!reverse) { range = RangeRequest.builder().startRowInclusive(row(0)).endRowExclusive(row(1)).build(); } else { range = RangeRequest.reverseBuilder().startRowInclusive(row(0)).build(); } ClosableIterator<RowResult<Set<Long>>> rangeWithHistory = keyValueService.getRangeOfTimestamps( TEST_TABLE, range, TEST_TIMESTAMP + 2); RowResult<Set<Long>> row = rangeWithHistory.next(); assertFalse(rangeWithHistory.hasNext()); rangeWithHistory.close(); assertEquals(1, Iterables.size(row.getCells())); Entry<Cell, Set<Long>> cell0 = row.getCells().iterator().next(); assertEquals(2, cell0.getValue().size()); assertTrue(cell0.getValue().contains(TEST_TIMESTAMP)); assertTrue(cell0.getValue().contains(TEST_TIMESTAMP + 1)); }
@SuppressWarnings("deprecation") private List<RawSqlRow> loadSqlRows(ConnectionSupplier conns) { List<RawSqlRow> sqlRows = new ArrayList<>(); try (ClosableIterator<AgnosticLightResultRow> rangeResults = selectNextPage(conns)) { while (rangeResults.hasNext()) { AgnosticLightResultRow row = rangeResults.next(); Cell cell = Cell.create(row.getBytes("row_name"), row.getBytes("col_name")); long ts = row.getLong("ts"); byte[] val = row.getBytes("val"); Long overflowId = haveOverflowValues ? row.getLongObject("overflow") : null; sqlRows.add(new RawSqlRow(cell, ts, val, overflowId)); } } return sqlRows; }
@Test public void testRangeBatchSizeOne() { RangeRequest range = RangeRequest.builder().batchHint(1).build(); ClosableIterator<RowResult<Value>> ranges = keyValueService.getRange(TEST_TABLE, range, 1); assertEquals(false, ranges.hasNext()); }
private byte[] getOnlyItemInTableRange() { try (ClosableIterator<RowResult<Value>> rangeIterator = keyValueService.getRange(TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 3)) { byte[] contents = rangeIterator.next().getOnlyColumnValue().getContents(); assertFalse("There should only be one row in the table", rangeIterator.hasNext()); return contents; } }
@Override protected BiMap<TableReference, TableReference> readTableMap() { BiMap<TableReference, TableReference> ret = HashBiMap.create(); ClosableIterator<RowResult<Value>> range = kv.getRange( AtlasDbConstants.NAMESPACE_TABLE, RangeRequest.builder().build(), Long.MAX_VALUE); try { while (range.hasNext()) { RowResult<Value> row = range.next(); String shortName = PtBytes.toString(row.getColumns() .get(AtlasDbConstants.NAMESPACE_SHORT_COLUMN_BYTES).getContents()); TableReference ref = getTableRefFromBytes(row.getRowName()); ret.put(ref, TableReference.createWithEmptyNamespace(shortName)); } } finally { range.close(); } return ret; }
int numSqlRows = 0; byte[] colName = null; while (iter.hasNext()) { numSqlRows += 1; AgnosticLightResultRow sqlRow = iter.next();
public static Long get(KeyValueService kvs, Long timeMillis) { byte[] row = EncodingUtils.encodeUnsignedVarLong(timeMillis); EncodingUtils.flipAllBitsInPlace(row); RangeRequest rangeRequest = RangeRequest.builder().startRowInclusive(row).batchHint(1).build(); try (ClosableIterator<RowResult<Value>> result = kvs.getRange(AtlasDbConstants.PUNCH_TABLE, rangeRequest, Long.MAX_VALUE)) { if (result.hasNext()) { return EncodingUtils.decodeUnsignedVarLong(result.next().getColumns().get(COLUMN).getContents()); } else { return Long.MIN_VALUE; } } }