protected void verifyMatchingResult(List<Map.Entry<Cell, byte[]>> expected, byte[] row, Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange) { assertEquals(1, columnRange.size()); assertArrayEquals(row, Iterables.getOnlyElement(columnRange.keySet())); BatchingVisitable<Map.Entry<Cell, byte[]>> batchingVisitable = Iterables.getOnlyElement(columnRange.values()); List<Map.Entry<Cell, byte[]>> results = BatchingVisitables.copyToList(batchingVisitable); assertEquals(expected.size(), results.size()); for (int i = 0 ; i < expected.size() ; i++) { assertEquals(expected.get(i).getKey(), results.get(i).getKey()); assertArrayEquals(expected.get(i).getValue(), results.get(i).getValue()); } }
private void verifyAllGetRangesImplsRangeSizes(Transaction t, RangeRequest templateRangeRequest, int expectedRangeSize) { Iterable<RangeRequest> rangeRequests = Iterables.limit(Iterables.cycle(templateRangeRequest), 1000); List<BatchingVisitable<RowResult<byte[]>>> getRangesWithPrefetchingImpl = ImmutableList.copyOf( t.getRanges(TEST_TABLE, rangeRequests)); List<BatchingVisitable<RowResult<byte[]>>> getRangesInParallelImpl = t.getRanges(TEST_TABLE, rangeRequests, 2, (rangeRequest, visitable) -> visitable).collect(Collectors.toList()); List<BatchingVisitable<RowResult<byte[]>>> getRangesLazyImpl = t.getRangesLazy(TEST_TABLE, rangeRequests).collect(Collectors.toList()); assertEquals(getRangesWithPrefetchingImpl.size(), getRangesLazyImpl.size()); assertEquals(getRangesLazyImpl.size(), getRangesInParallelImpl.size()); for (int i = 0; i < getRangesWithPrefetchingImpl.size(); i++) { assertEquals(expectedRangeSize, BatchingVisitables.copyToList(getRangesWithPrefetchingImpl.get(i)).size()); assertEquals(expectedRangeSize, BatchingVisitables.copyToList(getRangesInParallelImpl.get(i)).size()); assertEquals(expectedRangeSize, BatchingVisitables.copyToList(getRangesLazyImpl.get(i)).size()); } }
private List<RowResult<byte[]>> getRangeInner(ConsecutiveNarrowTable table) { final int rangeRequestSize = 1000; return table.getTransactionManager().runTaskThrowOnConflict(txn -> { RangeRequest request = Iterables.getOnlyElement(table.getRangeRequests(1, rangeRequestSize, false)); List<RowResult<byte[]>> results = BatchingVisitables.copyToList(txn.getRange( table.getTableRef(), request)); Preconditions.checkState(results.size() == rangeRequestSize, "Expected %s rows, found %s rows", rangeRequestSize, results.size()); return results; }); }
private List<SortedMap<Long, Multimap<TableReference, Cell>>> getScrubQueue() { BatchingVisitable<SortedMap<Long, Multimap<TableReference, Cell>>> visitable = scrubStore.getBatchingVisitableScrubQueue( Long.MAX_VALUE, PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY); return BatchingVisitables.copyToList(visitable); } }
private Iterable<BatchingVisitable<RowResult<byte[]>>> getRangesInner(ConsecutiveNarrowTable table) { return table.getTransactionManager().runTaskThrowOnConflict(txn -> { Iterable<RangeRequest> requests = table.getRangeRequests(1000, RANGES_SINGLE_REQUEST_SIZE, false); Iterable<BatchingVisitable<RowResult<byte[]>>> results = txn.getRanges(table.getTableRef(), requests); results.forEach(bvs -> { List<RowResult<byte[]>> result = BatchingVisitables.copyToList(bvs); Preconditions.checkState(result.size() == RANGES_SINGLE_REQUEST_SIZE, "Expected %s rows, found %s rows", RANGES_SINGLE_REQUEST_SIZE, result.size()); }); return results; }); }
private List<RowResult<byte[]>> getSingleRowWithRangeQueryInner(final ConsecutiveNarrowTable table) { return table.getTransactionManager().runTaskThrowOnConflict(txn -> { RangeRequest request = Iterables.getOnlyElement(table.getRangeRequests(1, 1, false)); List<RowResult<byte[]>> result = BatchingVisitables.copyToList( txn.getRange(table.getTableRef(), request)); byte[] rowName = Iterables.getOnlyElement(result).getRowName(); int rowNumber = ConsecutiveNarrowTable.rowNumber(rowName); int expectedRowNumber = ConsecutiveNarrowTable.rowNumber(request.getStartInclusive()); Preconditions.checkState(rowNumber == expectedRowNumber, "Start Row %s, row number %s", expectedRowNumber, rowNumber); return result; }); }
@Test public void testColumnSelection2() { String initialValue = "100"; Transaction t0 = startTransaction(); put(t0, "row1", "col1", initialValue); put(t0, "row1", "col2", initialValue); put(t0, "row2", "col1", initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(getRangeRetainingCol(t1, "col1")); BatchingVisitables.copyToList(getRangeRetainingCol(t1, "col2")); // We need to do at least one put so we don't get caught by the read only code path put(t1, "row22", "col2", initialValue); t1.commit(); }
@Test public void testRangesTransactionColumnSelection() { Transaction t = startTransaction(); put(t, "row1", "col1", "v1"); t.commit(); RangeRequest range1 = RangeRequest.builder().batchHint(3).build(); RangeRequest range2 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col1")))).build(); t = startTransaction(); Iterable<BatchingVisitable<RowResult<byte[]>>> ranges = t.getRanges(TEST_TABLE, Iterables.limit(Iterables.cycle(range1, range2), 1000)); for (BatchingVisitable<RowResult<byte[]>> batchingVisitable : ranges) { final List<RowResult<byte[]>> list = BatchingVisitables.copyToList(batchingVisitable); assertEquals(1, list.size()); assertEquals(1, list.get(0).getColumns().size()); } RangeRequest range3 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col2")))).build(); verifyAllGetRangesImplsRangeSizes(t, range3, 0); }
30L, ImmutableMultimap.of(foo, cell2), 40L, ImmutableMultimap.of(foo, cell3, bar, cell3, baz, cell3)), Iterables.getOnlyElement(BatchingVisitables.copyToList(visitable))); String output = new String(baos.toByteArray()); Assert.assertTrue(output, output.contains("Starting iteration 2"));
@Test public void testColumnSelection() { String initialValue = "100"; Transaction t0 = startTransaction(); String firstCol = "col1"; put(t0, "row1", firstCol, initialValue); put(t0, "row1", "col2", initialValue); put(t0, "row2", firstCol, initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(getRangeRetainingCol(t1, firstCol)); get(t1, "row1", "col2"); // We need to do at least one put so we don't get caught by the read only code path put(t1, "row22", "col2", initialValue); t1.commit(); }
@Test public void testScrubQueueIsCleared() { Cell cell1 = Cell.create(new byte[] {1}, new byte[] {2}); Cell cell2 = Cell.create(new byte[] {2}, new byte[] {3}); Cell cell3 = Cell.create(new byte[] {3}, new byte[] {4}); TableReference tableRef = TableReference.createFromFullyQualifiedName("foo.bar"); kvs.createTable(tableRef, new byte[] {}); kvs.putWithTimestamps(tableRef, ImmutableMultimap.<Cell, Value>builder() .put(cell1, Value.create(new byte[] {3}, 10)) .put(cell1, Value.create(new byte[] {4}, 20)) .put(cell2, Value.create(new byte[] {4}, 30)) .put(cell2, Value.create(new byte[] {5}, 40)) .put(cell2, Value.create(new byte[] {6}, 50)) .put(cell3, Value.create(new byte[] {7}, 60)) .build()); transactions.putUnlessExists(10, 15); transactions.putUnlessExists(20, 25); transactions.putUnlessExists(30, 35); transactions.putUnlessExists(50, 55); transactions.putUnlessExists(60, 65); scrubStore.queueCellsForScrubbing(ImmutableMultimap.of(cell1, tableRef), 10, 100); scrubStore.queueCellsForScrubbing(ImmutableMultimap.of(cell1, tableRef), 20, 100); scrubStore.queueCellsForScrubbing(ImmutableMultimap.of(cell2, tableRef), 40, 100); scrubStore.queueCellsForScrubbing(ImmutableMultimap.of(cell2, tableRef), 50, 100); scrubStore.queueCellsForScrubbing(ImmutableMultimap.of(cell3, tableRef), 60, 100); scrubber.runBackgroundScrubTask(null); List<SortedMap<Long, Multimap<TableReference, Cell>>> scrubQueue = BatchingVisitables.copyToList( scrubStore.getBatchingVisitableScrubQueue(Long.MAX_VALUE, null, null)); Assert.assertEquals(ImmutableList.of(), scrubQueue); }
@Test public void testCellReadWriteFailure2() { String initialValue = "100"; Transaction t0 = startTransaction(); put(t0, "row1", "col1", initialValue); put(t0, "row2", "col1", initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(t1.getRange(TEST_TABLE, RangeRequest.builder().build())); put(t1, "row22", "col1", initialValue); Transaction t2 = startTransaction(); put(t2, "row2", "col1", "101"); t2.commit(); try { t1.commit(); fail(); } catch (TransactionSerializableConflictException e) { // this is expectecd to throw because it is a write skew } }
@Test public void testPhantomReadFail2() { String initialValue = "100"; Transaction t0 = startTransaction(); put(t0, "row1", "col1", initialValue); put(t0, "row2", "col1", initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(t1.getRange(TEST_TABLE, RangeRequest.builder().build())); put(t1, "row22", "col1", initialValue); Transaction t2 = startTransaction(); put(t2, "row3", "col1", initialValue); t2.commit(); try { t1.commit(); fail(); } catch (TransactionSerializableConflictException e) { // this is expectecd to throw because it is a write skew } }
@Test public void testCellReadWriteFailure() { String initialValue = "100"; Transaction t0 = startTransaction(); put(t0, "row1", "col1", initialValue); put(t0, "row2", "col1", initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(t1.getRange(TEST_TABLE, RangeRequest.builder().build())); put(t1, "row22", "col1", initialValue); Transaction t2 = startTransaction(); put(t2, "row3", "col1", initialValue); t2.commit(); try { t1.commit(); fail(); } catch (TransactionSerializableConflictException e) { // this is expectecd to throw because it is a write skew } }
protected void verifyMatchingResult(List<Map.Entry<Cell, byte[]>> expected, byte[] row, Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> columnRange) { assertEquals(1, columnRange.size()); assertArrayEquals(row, Iterables.getOnlyElement(columnRange.keySet())); BatchingVisitable<Map.Entry<Cell, byte[]>> batchingVisitable = Iterables.getOnlyElement(columnRange.values()); List<Map.Entry<Cell, byte[]>> results = BatchingVisitables.copyToList(batchingVisitable); assertEquals(expected.size(), results.size()); for (int i = 0 ; i < expected.size() ; i++) { assertEquals(expected.get(i).getKey(), results.get(i).getKey()); assertArrayEquals(expected.get(i).getValue(), results.get(i).getValue()); } }
private void verifyAllGetRangesImplsRangeSizes(Transaction t, RangeRequest templateRangeRequest, int expectedRangeSize) { Iterable<RangeRequest> rangeRequests = Iterables.limit(Iterables.cycle(templateRangeRequest), 1000); List<BatchingVisitable<RowResult<byte[]>>> getRangesWithPrefetchingImpl = ImmutableList.copyOf( t.getRanges(TEST_TABLE, rangeRequests)); List<BatchingVisitable<RowResult<byte[]>>> getRangesInParallelImpl = t.getRanges(TEST_TABLE, rangeRequests, 2, (rangeRequest, visitable) -> visitable).collect(Collectors.toList()); List<BatchingVisitable<RowResult<byte[]>>> getRangesLazyImpl = t.getRangesLazy(TEST_TABLE, rangeRequests).collect(Collectors.toList()); assertEquals(getRangesWithPrefetchingImpl.size(), getRangesLazyImpl.size()); assertEquals(getRangesLazyImpl.size(), getRangesInParallelImpl.size()); for (int i = 0; i < getRangesWithPrefetchingImpl.size(); i++) { assertEquals(expectedRangeSize, BatchingVisitables.copyToList(getRangesWithPrefetchingImpl.get(i)).size()); assertEquals(expectedRangeSize, BatchingVisitables.copyToList(getRangesInParallelImpl.get(i)).size()); assertEquals(expectedRangeSize, BatchingVisitables.copyToList(getRangesLazyImpl.get(i)).size()); } }
@Test public void testColumnSelection2() { String initialValue = "100"; Transaction t0 = startTransaction(); put(t0, "row1", "col1", initialValue); put(t0, "row1", "col2", initialValue); put(t0, "row2", "col1", initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(getRangeRetainingCol(t1, "col1")); BatchingVisitables.copyToList(getRangeRetainingCol(t1, "col2")); // We need to do at least one put so we don't get caught by the read only code path put(t1, "row22", "col2", initialValue); t1.commit(); }
@Test public void testRangesTransactionColumnSelection() { Transaction t = startTransaction(); put(t, "row1", "col1", "v1"); t.commit(); RangeRequest range1 = RangeRequest.builder().batchHint(3).build(); RangeRequest range2 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col1")))).build(); t = startTransaction(); Iterable<BatchingVisitable<RowResult<byte[]>>> ranges = t.getRanges(TEST_TABLE, Iterables.limit(Iterables.cycle(range1, range2), 1000)); for (BatchingVisitable<RowResult<byte[]>> batchingVisitable : ranges) { final List<RowResult<byte[]>> list = BatchingVisitables.copyToList(batchingVisitable); assertEquals(1, list.size()); assertEquals(1, list.get(0).getColumns().size()); } RangeRequest range3 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col2")))).build(); verifyAllGetRangesImplsRangeSizes(t, range3, 0); }
@Test public void testColumnSelection() { String initialValue = "100"; Transaction t0 = startTransaction(); String firstCol = "col1"; put(t0, "row1", firstCol, initialValue); put(t0, "row1", "col2", initialValue); put(t0, "row2", firstCol, initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(getRangeRetainingCol(t1, firstCol)); get(t1, "row1", "col2"); // We need to do at least one put so we don't get caught by the read only code path put(t1, "row22", "col2", initialValue); t1.commit(); }
@Test public void testCellReadWriteFailure() { String initialValue = "100"; Transaction t0 = startTransaction(); put(t0, "row1", "col1", initialValue); put(t0, "row2", "col1", initialValue); t0.commit(); Transaction t1 = startTransaction(); BatchingVisitables.copyToList(t1.getRange(TEST_TABLE, RangeRequest.builder().build())); put(t1, "row22", "col1", initialValue); Transaction t2 = startTransaction(); put(t2, "row3", "col1", initialValue); t2.commit(); try { t1.commit(); fail(); } catch (TransactionSerializableConflictException e) { // this is expectecd to throw because it is a write skew } }