@Test(timeout = 50000) public void testDontSweepLatestConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(0, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("bar", getFromDefaultColumn("foo", 150)); assertEqualsDisregardingExtraSentinels(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepLatestDeletedThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertNull(getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(), getAllTsFromDefaultColumn("foo")); }
private void testSweepManyRows(SweepStrategy strategy) { createTable(strategy); putIntoDefaultColumn("foo", "bar1", 5); putIntoDefaultColumn("foo", "bar2", 10); putIntoDefaultColumn("baz", "bar3", 15); putIntoDefaultColumn("baz", "bar4", 20); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(2, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(4); }); }
@Test(timeout = 50000) public void testDontSweepLatestThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(0, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("bar", getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
static SweepTableResponse from(SweepResults results) { return ImmutableSweepTableResponse.builder() .numCellTsPairsExamined(results.getCellTsPairsExamined()) .staleValuesDeleted(results.getStaleValuesDeleted()) .nextStartRow(results.getNextStartRow().map(PtBytes::encodeHexString)) .build(); }
@Test(timeout = 50000) public void testSweepUncommittedThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "baz", 100); SweepResults results = completeSweep(175).get(); assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); assertEquals("bar", getFromDefaultColumn("foo", 750)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepUncommittedConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "baz", 100); SweepResults results = completeSweep(175).get(); assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); assertEquals("bar", getFromDefaultColumn("foo", 750)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepOnMixedCaseTable() { TableReference mixedCaseTable = TableReference.create(Namespace.create("someNamespace"), "someTable"); createTable(mixedCaseTable, SweepStrategy.CONSERVATIVE); put(mixedCaseTable, "row", "col", "val", 10); put(mixedCaseTable, "row", "col", "val", 20); Optional<SweepResults> optResults = completeSweep(mixedCaseTable, 30); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); }
@Test(timeout = 50000) public void testSweepLatestDeletedMultiValConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "old value", 40); putIntoDefaultColumn("foo", "", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("", getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(-1L, 50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepLatestNotDeletedMultiValThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "old value", 40); putIntoDefaultColumn("foo", "new value", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("new value", getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepLatestDeletedMultiRowThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "", 50); put("foo-2", "other", "womp", 60); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertNull(getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepManyLatestDeletedThorough2() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "foo", 125); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(2, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(3); }); assertEquals("foo", getFromDefaultColumn("foo", 200)); assertEquals(ImmutableSet.of(125L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepOneConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo", "baz", 100); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); assertEquals("baz", getFromDefaultColumn("foo", 150)); assertEquals("", getFromDefaultColumn("foo", 80)); assertEquals(ImmutableSet.of(-1L, 100L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepOneThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo", "baz", 100); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); assertEquals("baz", getFromDefaultColumn("foo", 150)); assertNull(getFromDefaultColumn("foo", 80)); assertEquals(ImmutableSet.of(100L), getAllTsFromDefaultColumn("foo")); }
/** * Returns a new {@link SweepResults} representing cumulative results from this instance and {@code other}. * The operation is commutative. */ public SweepResults accumulateWith(SweepResults other) { return SweepResults.builder() .nextStartRow(maxRowOptional(getNextStartRow(), other.getNextStartRow())) .cellTsPairsExamined(getCellTsPairsExamined() + other.getCellTsPairsExamined()) .staleValuesDeleted(getStaleValuesDeleted() + other.getStaleValuesDeleted()) .minSweptTimestamp(Math.min(getMinSweptTimestamp(), other.getMinSweptTimestamp())) .timeInMillis(getTimeInMillis() + other.getTimeInMillis()) .timeSweepStarted(Math.min(getTimeSweepStarted(), other.getTimeSweepStarted())) .build(); }
@Test(timeout = 50000) public void testSweepManyLatestDeletedThoroughIncludingUncommitted2() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "foo", 125); putUncommitted("foo", "", 150); SweepResults results = completeSweep(175).get(); assertEquals(4, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(5); assertEquals("foo", getFromDefaultColumn("foo", 200)); assertEquals(ImmutableSet.of(125L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepManyValuesThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "buzz", 125); putUncommitted("foo", "foo", 150); SweepResults results = completeSweep(175).get(); assertEquals(4, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(5); assertEquals("buzz", getFromDefaultColumn("foo", 200)); assertNull(getFromDefaultColumn("foo", 124)); assertEquals(ImmutableSet.of(125L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepManyValuesIncludingUncommittedConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "buzz", 125); putUncommitted("foo", "foo", 150); SweepResults results = completeSweep(175).get(); assertEquals(4, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(5); assertEquals("buzz", getFromDefaultColumn("foo", 200)); assertEquals("", getFromDefaultColumn("foo", 124)); assertEquals(ImmutableSet.of(-1L, 125L), getAllTsFromDefaultColumn("foo")); }
private void processFinishedSweep(TableToSweep tableToSweep, SweepResults cumulativeResults) { saveFinalSweepResults(tableToSweep, cumulativeResults); log.info("Finished sweeping table {}. Examined {} cell+timestamp pairs, deleted {} stale values. Time taken " + "sweeping: {} ms, time elapsed since sweep first started on this table: {} ms.", LoggingArgs.tableRef("tableRef", tableToSweep.getTableRef()), SafeArg.of("cellTs pairs examined", cumulativeResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", cumulativeResults.getStaleValuesDeleted()), SafeArg.of("time sweeping table", cumulativeResults.getTimeInMillis()), SafeArg.of("time elapsed", cumulativeResults.getTimeElapsedSinceStartedSweeping())); tableToSweep.getSweepLock().close(); sweepProgressStore.clearProgress(tableToSweep.getTableRef()); }
private void saveFinalSweepResults(TableToSweep tableToSweep, SweepResults finalSweepResults) { txManager.runTaskWithRetry((TxTask) tx -> { ImmutableUpdateSweepPriority.Builder update = ImmutableUpdateSweepPriority.builder() .newStaleValuesDeleted(finalSweepResults.getStaleValuesDeleted()) .newCellTsPairsExamined(finalSweepResults.getCellTsPairsExamined()) .newLastSweepTimeMillis(wallClock.getTimeMillis()) .newMinimumSweptTimestamp(finalSweepResults.getMinSweptTimestamp()); if (!tableToSweep.hasPreviousProgress()) { // This is the first (and only) set of results being written for this table. update.newWriteCount(0L); } sweepPriorityStore.update(tx, tableToSweep.getTableRef(), update.build()); return null; }); }