@Test public void shouldReturnValuesForMultipleColumnsWhenSweeping() { createTable(TableMetadataPersistence.SweepStrategy.CONSERVATIVE); for (int ts = 10; ts <= 150; ts += 10) { put("row", "col1", "value", ts); put("row", "col2", "value", ts + 5); } SweepResults results = completeSweep(350).get(); Assert.assertEquals(28, results.getStaleValuesDeleted()); }
@Test(timeout = 50000) public void testDontSweepLatestThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(0, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("bar", getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
static SweepTableResponse from(SweepResults results) { return ImmutableSweepTableResponse.builder() .numCellTsPairsExamined(results.getCellTsPairsExamined()) .staleValuesDeleted(results.getStaleValuesDeleted()) .nextStartRow(results.getNextStartRow().map(PtBytes::encodeHexString)) .build(); }
@Test(timeout = 50000) public void testDontSweepLatestConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(0, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("bar", getFromDefaultColumn("foo", 150)); assertEqualsDisregardingExtraSentinels(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepLatestDeletedThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertNull(getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(), getAllTsFromDefaultColumn("foo")); }
private void testSweepManyRows(SweepStrategy strategy) { createTable(strategy); putIntoDefaultColumn("foo", "bar1", 5); putIntoDefaultColumn("foo", "bar2", 10); putIntoDefaultColumn("baz", "bar3", 15); putIntoDefaultColumn("baz", "bar4", 20); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(2, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(4); }); }
@Test(timeout = 50000) public void testSweepUncommittedThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "baz", 100); SweepResults results = completeSweep(175).get(); assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); assertEquals("bar", getFromDefaultColumn("foo", 750)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepUncommittedConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "baz", 100); SweepResults results = completeSweep(175).get(); assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); assertEquals("bar", getFromDefaultColumn("foo", 750)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepOnMixedCaseTable() { TableReference mixedCaseTable = TableReference.create(Namespace.create("someNamespace"), "someTable"); createTable(mixedCaseTable, SweepStrategy.CONSERVATIVE); put(mixedCaseTable, "row", "col", "val", 10); put(mixedCaseTable, "row", "col", "val", 20); Optional<SweepResults> optResults = completeSweep(mixedCaseTable, 30); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); }
@Test(timeout = 50000) public void testSweepLatestDeletedMultiValConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "old value", 40); putIntoDefaultColumn("foo", "", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("", getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(-1L, 50L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepLatestDeletedMultiRowThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "", 50); put("foo-2", "other", "womp", 60); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertNull(getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepLatestNotDeletedMultiValThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "old value", 40); putIntoDefaultColumn("foo", "new value", 50); Optional<SweepResults> optResults = completeSweep(75); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); assertEquals("new value", getFromDefaultColumn("foo", 150)); assertEquals(ImmutableSet.of(50L), getAllTsFromDefaultColumn("foo")); }
private Object runSingleSweep(RegeneratingTable table, int uniqueCellsToSweep) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT * uniqueCellsToSweep) .candidateBatchSize(RegeneratingTable.SWEEP_DUPLICATES * uniqueCellsToSweep + 1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES * uniqueCellsToSweep) .build(); SweepResults sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, PtBytes.EMPTY_BYTE_ARRAY); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT * uniqueCellsToSweep)); return sweepResults; }
@Test(timeout = 50000) public void testSweepManyLatestDeletedThorough2() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "foo", 125); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(2, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(3); }); assertEquals("foo", getFromDefaultColumn("foo", 200)); assertEquals(ImmutableSet.of(125L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepOneConservative() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo", "baz", 100); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); assertEquals("baz", getFromDefaultColumn("foo", 150)); assertEquals("", getFromDefaultColumn("foo", 80)); assertEquals(ImmutableSet.of(-1L, 100L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweepOneThorough() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo", "baz", 100); Optional<SweepResults> optResults = completeSweep(175); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); assertEquals("baz", getFromDefaultColumn("foo", 150)); assertNull(getFromDefaultColumn("foo", 80)); assertEquals(ImmutableSet.of(100L), getAllTsFromDefaultColumn("foo")); }
/** * Returns a new {@link SweepResults} representing cumulative results from this instance and {@code other}. * The operation is commutative. */ public SweepResults accumulateWith(SweepResults other) { return SweepResults.builder() .nextStartRow(maxRowOptional(getNextStartRow(), other.getNextStartRow())) .cellTsPairsExamined(getCellTsPairsExamined() + other.getCellTsPairsExamined()) .staleValuesDeleted(getStaleValuesDeleted() + other.getStaleValuesDeleted()) .minSweptTimestamp(Math.min(getMinSweptTimestamp(), other.getMinSweptTimestamp())) .timeInMillis(getTimeInMillis() + other.getTimeInMillis()) .timeSweepStarted(Math.min(getTimeSweepStarted(), other.getTimeSweepStarted())) .build(); }
@Test(timeout = 50000) public void testSweepManyLatestDeletedThoroughIncludingUncommitted2() { createTable(SweepStrategy.THOROUGH); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "foo", 125); putUncommitted("foo", "", 150); SweepResults results = completeSweep(175).get(); assertEquals(4, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(5); assertEquals("foo", getFromDefaultColumn("foo", 200)); assertEquals(ImmutableSet.of(125L), getAllTsFromDefaultColumn("foo")); }
@Test public void should_not_oom_when_there_are_many_large_values_to_sweep() { createTable(TableMetadataPersistence.SweepStrategy.CONSERVATIVE); long numInsertions = 100; insertMultipleValues(numInsertions); long sweepTimestamp = numInsertions + 1; SweepResults results = completeSweep(sweepTimestamp).get(); Assert.assertEquals(numInsertions - 1, results.getStaleValuesDeleted()); }
private void processFinishedSweep(TableToSweep tableToSweep, SweepResults cumulativeResults) { saveFinalSweepResults(tableToSweep, cumulativeResults); log.info("Finished sweeping table {}. Examined {} cell+timestamp pairs, deleted {} stale values. Time taken " + "sweeping: {} ms, time elapsed since sweep first started on this table: {} ms.", LoggingArgs.tableRef("tableRef", tableToSweep.getTableRef()), SafeArg.of("cellTs pairs examined", cumulativeResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", cumulativeResults.getStaleValuesDeleted()), SafeArg.of("time sweeping table", cumulativeResults.getTimeInMillis()), SafeArg.of("time elapsed", cumulativeResults.getTimeElapsedSinceStartedSweeping())); tableToSweep.getSweepLock().close(); sweepProgressStore.clearProgress(tableToSweep.getTableRef()); }