public byte[] getStartRow() { return previousResults.getNextStartRow().orElse(PtBytes.EMPTY_BYTE_ARRAY); }
private SweepResults runFullSweepWithoutSavingResults( TableReference tableRef, byte[] startRow, SweepBatchConfig sweepBatchConfig) { SweepResults cumulativeResults = SweepResults.createEmptySweepResult(Optional.of(startRow)); while (cumulativeResults.getNextStartRow().isPresent()) { SweepResults results = runOneBatchWithoutSavingResults( tableRef, cumulativeResults.getNextStartRow().get(), sweepBatchConfig); specificTableSweeper.updateTimeMetricsOneIteration(results.getTimeInMillis(), results.getTimeElapsedSinceStartedSweeping()); cumulativeResults = cumulativeResults.accumulateWith(results); } return cumulativeResults; }
private SweepResults partialSweep(long ts) { sweepTimestamp.set(ts); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(1) .candidateBatchSize(1) .maxCellTsPairsToExamine(1) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertTrue(results.getNextStartRow().isPresent()); return results; } }
private void processSweepResults(TableToSweep tableToSweep, SweepResults currentIteration) { updateTimeMetricsOneIteration(currentIteration.getTimeInMillis(), currentIteration.getTimeElapsedSinceStartedSweeping()); SweepResults cumulativeResults = getCumulativeSweepResults(tableToSweep, currentIteration); if (currentIteration.getNextStartRow().isPresent()) { saveIntermediateSweepResults(tableToSweep, cumulativeResults); } else { processFinishedSweep(tableToSweep, cumulativeResults); } }
static SweepTableResponse from(SweepResults results) { return ImmutableSweepTableResponse.builder() .numCellTsPairsExamined(results.getCellTsPairsExamined()) .staleValuesDeleted(results.getStaleValuesDeleted()) .nextStartRow(results.getNextStartRow().map(PtBytes::encodeHexString)) .build(); }
private Object runMultiSweep(RegeneratingTable table) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepResults sweepResults = null; byte[] nextStartRow = PtBytes.EMPTY_BYTE_ARRAY; for (int i = 0; i < BATCH_SIZE; i++) { SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT) .candidateBatchSize(1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES) .build(); sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, nextStartRow); nextStartRow = sweepResults.getNextStartRow().get(); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT)); } return sweepResults; }
totalStaleValuesDeleted += results.getStaleValuesDeleted(); totalCellsExamined += results.getCellTsPairsExamined(); if (!results.getNextStartRow().isPresent()) { return Optional.of(SweepResults.builder() .staleValuesDeleted(totalStaleValuesDeleted) .build()); startRow = results.getNextStartRow().get();
/** * Returns a new {@link SweepResults} representing cumulative results from this instance and {@code other}. * The operation is commutative. */ public SweepResults accumulateWith(SweepResults other) { return SweepResults.builder() .nextStartRow(maxRowOptional(getNextStartRow(), other.getNextStartRow())) .cellTsPairsExamined(getCellTsPairsExamined() + other.getCellTsPairsExamined()) .staleValuesDeleted(getStaleValuesDeleted() + other.getStaleValuesDeleted()) .minSweptTimestamp(Math.min(getMinSweptTimestamp(), other.getMinSweptTimestamp())) .timeInMillis(getTimeInMillis() + other.getTimeInMillis()) .timeSweepStarted(Math.min(getTimeSweepStarted(), other.getTimeSweepStarted())) .build(); }
while (accumulatedResults.getNextStartRow().isPresent()) { SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()); + " {} ms. Total time sweeping this table: {} ms.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), UnsafeArg.of("startRow", encodeStartRow(accumulatedResults.getNextStartRow())), UnsafeArg.of("exclusiveEndRow", encodeEndRow(newResults.getNextStartRow())), LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken millis", newResults.getTimeInMillis()),
@Test(timeout = 50000) public void testSweeperFailsHalfwayThroughOnDeleteTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo2", "bang", 75); putIntoDefaultColumn("foo3", "baz", 100); putIntoDefaultColumn("foo4", "buzz", 125); byte[] nextStartRow = partialSweep(150).getNextStartRow().get(); kvs.dropTable(TABLE_NAME); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), nextStartRow); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); }
private void saveIntermediateSweepResults(TableToSweep tableToSweep, SweepResults results) { Preconditions.checkArgument(results.getNextStartRow().isPresent(), "Next start row should be present when saving intermediate results!"); txManager.runTaskWithRetry((TxTask) tx -> { if (!tableToSweep.hasPreviousProgress()) { // This is the first set of results being written for this table. sweepPriorityStore.update( tx, tableToSweep.getTableRef(), ImmutableUpdateSweepPriority.builder().newWriteCount(0L).build()); } SweepProgress newProgress = ImmutableSweepProgress.builder() .tableRef(tableToSweep.getTableRef()) .staleValuesDeleted(results.getStaleValuesDeleted()) .cellTsPairsExamined(results.getCellTsPairsExamined()) //noinspection OptionalGetWithoutIsPresent // covered by precondition above .startRow(results.getNextStartRow().get()) .startColumn(PtBytes.toBytes("unused")) .minimumSweptTimestamp(results.getMinSweptTimestamp()) .timeInMillis(results.getTimeInMillis()) .startTimeInMillis(results.getTimeSweepStarted()) .build(); sweepProgressStore.saveProgress(newProgress); return null; }); }
public byte[] getStartRow() { return previousResults.getNextStartRow().orElse(PtBytes.EMPTY_BYTE_ARRAY); }
private SweepResults runFullSweepWithoutSavingResults( TableReference tableRef, byte[] startRow, SweepBatchConfig sweepBatchConfig) { SweepResults cumulativeResults = SweepResults.createEmptySweepResult(Optional.of(startRow)); while (cumulativeResults.getNextStartRow().isPresent()) { SweepResults results = runOneBatchWithoutSavingResults( tableRef, cumulativeResults.getNextStartRow().get(), sweepBatchConfig); specificTableSweeper.updateTimeMetricsOneIteration(results.getTimeInMillis(), results.getTimeElapsedSinceStartedSweeping()); cumulativeResults = cumulativeResults.accumulateWith(results); } return cumulativeResults; }
private void processSweepResults(TableToSweep tableToSweep, SweepResults currentIteration) { updateTimeMetricsOneIteration(currentIteration.getTimeInMillis(), currentIteration.getTimeElapsedSinceStartedSweeping()); SweepResults cumulativeResults = getCumulativeSweepResults(tableToSweep, currentIteration); if (currentIteration.getNextStartRow().isPresent()) { saveIntermediateSweepResults(tableToSweep, cumulativeResults); } else { processFinishedSweep(tableToSweep, cumulativeResults); } }
private SweepResults partialSweep(long ts) { sweepTimestamp.set(ts); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(1) .candidateBatchSize(1) .maxCellTsPairsToExamine(1) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertTrue(results.getNextStartRow().isPresent()); return results; } }
static SweepTableResponse from(SweepResults results) { return ImmutableSweepTableResponse.builder() .numCellTsPairsExamined(results.getCellTsPairsExamined()) .staleValuesDeleted(results.getStaleValuesDeleted()) .nextStartRow(results.getNextStartRow().map(PtBytes::encodeHexString)) .build(); }
/** * Returns a new {@link SweepResults} representing cumulative results from this instance and {@code other}. * The operation is commutative. */ public SweepResults accumulateWith(SweepResults other) { return SweepResults.builder() .nextStartRow(maxRowOptional(getNextStartRow(), other.getNextStartRow())) .cellTsPairsExamined(getCellTsPairsExamined() + other.getCellTsPairsExamined()) .staleValuesDeleted(getStaleValuesDeleted() + other.getStaleValuesDeleted()) .minSweptTimestamp(Math.min(getMinSweptTimestamp(), other.getMinSweptTimestamp())) .timeInMillis(getTimeInMillis() + other.getTimeInMillis()) .timeSweepStarted(Math.min(getTimeSweepStarted(), other.getTimeSweepStarted())) .build(); }
totalStaleValuesDeleted += results.getStaleValuesDeleted(); totalCellsExamined += results.getCellTsPairsExamined(); if (!results.getNextStartRow().isPresent()) { return Optional.of(SweepResults.builder() .staleValuesDeleted(totalStaleValuesDeleted) .build()); startRow = results.getNextStartRow().get();
@Test(timeout = 50000) public void testSweeperFailsHalfwayThroughOnDeleteTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo2", "bang", 75); putIntoDefaultColumn("foo3", "baz", 100); putIntoDefaultColumn("foo4", "buzz", 125); byte[] nextStartRow = partialSweep(150).getNextStartRow().get(); kvs.dropTable(TABLE_NAME); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), nextStartRow); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); }
private void saveIntermediateSweepResults(TableToSweep tableToSweep, SweepResults results) { Preconditions.checkArgument(results.getNextStartRow().isPresent(), "Next start row should be present when saving intermediate results!"); txManager.runTaskWithRetry((TxTask) tx -> { if (!tableToSweep.hasPreviousProgress()) { // This is the first set of results being written for this table. sweepPriorityStore.update( tx, tableToSweep.getTableRef(), ImmutableUpdateSweepPriority.builder().newWriteCount(0L).build()); } SweepProgress newProgress = ImmutableSweepProgress.builder() .tableRef(tableToSweep.getTableRef()) .staleValuesDeleted(results.getStaleValuesDeleted()) .cellTsPairsExamined(results.getCellTsPairsExamined()) //noinspection OptionalGetWithoutIsPresent // covered by precondition above .startRow(results.getNextStartRow().get()) .startColumn(PtBytes.toBytes("unused")) .minimumSweptTimestamp(results.getMinSweptTimestamp()) .timeInMillis(results.getTimeInMillis()) .startTimeInMillis(results.getTimeSweepStarted()) .build(); sweepProgressStore.saveProgress(newProgress); return null; }); }