/** * Filters out all writes made into tables with SweepStrategy NOTHING, then partitions the writes according to * shard, strategy, and start timestamp of the transaction that performed the write. */ public Map<PartitionInfo, List<WriteInfo>> filterAndPartition(List<WriteInfo> writes) { return partitionWritesByShardStrategyTimestamp(filterOutUnsweepableTables(writes)); }
private boolean isConservative(WriteInfo write) { return getStrategy(write) == TableMetadataPersistence.SweepStrategy.CONSERVATIVE; } }
public void enqueue(List<WriteInfo> allWrites) { Map<Cell, byte[]> referencesToDedicatedCells = new HashMap<>(); Map<Cell, byte[]> cellsToWrite = new HashMap<>(); Map<PartitionInfo, List<WriteInfo>> partitionedWrites = partitioner.filterAndPartition(allWrites); partitionedWrites.forEach((partitionInfo, writes) -> { referencesToDedicatedCells.putAll(populateReferences(partitionInfo, writes)); cellsToWrite.putAll(populateCells(partitionInfo, writes)); }); partitionedWrites.keySet().stream() .map(PartitionInfo::timestamp) .mapToLong(x -> x) .max() .ifPresent(timestamp -> { write(referencesToDedicatedCells, timestamp); write(cellsToWrite, timestamp); updateWriteMetrics(partitionedWrites); }); }
@Before public void setup() { partitioner = new WriteInfoPartitioner(mockKvs, () -> numShards); when(mockKvs.getMetadataForTable(any(TableReference.class))).thenAnswer(args -> METADATA_MAP.getOrDefault(args.getArguments()[0], AtlasDbConstants.EMPTY_TABLE_METADATA)); }
@Test public void partitionWritesByShardStrategyTimestampGroupsOnShardClash() { List<WriteInfo> writes = new ArrayList<>(); for (int i = 0; i <= numShards; i++) { writes.add(getWriteInfoWithFixedCellHash(CONSERVATIVE, i)); } Map<PartitionInfo, List<WriteInfo>> partitions = partitioner.partitionWritesByShardStrategyTimestamp(writes); assertThat(partitions.keySet()) .containsExactly(PartitionInfo.of(writes.get(0).toShard(numShards), true, 1L)); assertThat(Iterables.getOnlyElement(partitions.values())).isEqualTo(writes); }
@Test public void filterOutUnsweepableRemovesWritesWithStrategyNothing() { List<WriteInfo> writes = ImmutableList.of( getWriteInfoWithFixedCellHash(CONSERVATIVE, 0), getWriteInfoWithFixedCellHash(NOTHING, 1), getWriteInfoWithFixedCellHash(CONSERVATIVE, 0), getWriteInfoWithFixedCellHash(CONSERVATIVE2, 1), getWriteInfoWithFixedCellHash(THOROUGH, 2), getWriteInfoWithFixedCellHash(NOTHING, 0)); assertThat(partitioner.filterOutUnsweepableTables(writes)) .containsExactly( getWriteInfoWithFixedCellHash(CONSERVATIVE, 0), getWriteInfoWithFixedCellHash(CONSERVATIVE, 0), getWriteInfoWithFixedCellHash(CONSERVATIVE2, 1), getWriteInfoWithFixedCellHash(THOROUGH, 2)); }
@VisibleForTesting Map<PartitionInfo, List<WriteInfo>> partitionWritesByShardStrategyTimestamp(List<WriteInfo> writes) { int shards = numShards.get(); return writes.stream() .collect(Collectors.groupingBy(write -> getPartitionInfo(write, shards), Collectors.toList())); }
private PartitionInfo getPartitionInfo(WriteInfo write, int shards) { return PartitionInfo.of(write.toShard(shards), isConservative(write), write.timestamp()); }
static SweepQueueFactory create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction) { Schemas.createTablesAndIndexes(TargetedSweepSchema.INSTANCE.getLatestSchema(), kvs); ShardProgress shardProgress = new ShardProgress(kvs); Supplier<Integer> shards = createProgressUpdatingSupplier(shardsConfig, shardProgress, SweepQueueUtils.REFRESH_TIME); WriteInfoPartitioner partitioner = new WriteInfoPartitioner(kvs, shards); SweepableCells cells = new SweepableCells(kvs, partitioner, metrics, transaction); SweepableTimestamps timestamps = new SweepableTimestamps(kvs, partitioner); return new SweepQueueFactory(shardProgress, shards, cells, timestamps, metrics, kvs, timelock); }
@Test public void changingNumberOfPartitionsIsReflectedInPartitionInfo() { WriteInfo write = getWriteInfo(CONSERVATIVE, 1, 1, 100L); PartitionInfo partition1 = Iterables.getOnlyElement( partitioner.partitionWritesByShardStrategyTimestamp(ImmutableList.of(write)).keySet()); numShards += 1; PartitionInfo partition2 = Iterables.getOnlyElement( partitioner.partitionWritesByShardStrategyTimestamp(ImmutableList.of(write)).keySet()); assertThat(partition1.isConservative()).isEqualTo(partition2.isConservative()); assertThat(partition1.timestamp()).isEqualTo(partition2.timestamp()); assertThat(partition1.shard()).isNotEqualTo(partition2.shard()); }
@VisibleForTesting Map<PartitionInfo, List<WriteInfo>> partitionWritesByShardStrategyTimestamp(List<WriteInfo> writes) { int shards = numShards.get(); return writes.stream() .collect(Collectors.groupingBy(write -> getPartitionInfo(write, shards), Collectors.toList())); }
private PartitionInfo getPartitionInfo(WriteInfo write, int shards) { return PartitionInfo.of(write.toShard(shards), isConservative(write), write.timestamp()); }
/** * Filters out all writes made into tables with SweepStrategy NOTHING, then partitions the writes according to * shard, strategy, and start timestamp of the transaction that performed the write. */ public Map<PartitionInfo, List<WriteInfo>> filterAndPartition(List<WriteInfo> writes) { return partitionWritesByShardStrategyTimestamp(filterOutUnsweepableTables(writes)); }
@VisibleForTesting List<WriteInfo> filterOutUnsweepableTables(List<WriteInfo> writes) { return writes.stream() .filter(writeInfo -> getStrategy(writeInfo) != TableMetadataPersistence.SweepStrategy.NOTHING) .collect(Collectors.toList()); }
@Before public void setup() { numShards = DEFAULT_SHARDS; unreadableTs = SweepQueueUtils.TS_COARSE_GRANULARITY * 5; immutableTs = SweepQueueUtils.TS_COARSE_GRANULARITY * 5; metricsManager = MetricsManagers.createForTests(); timestampsSupplier = new SpecialTimestampsSupplier(() -> unreadableTs, () -> immutableTs); spiedKvs = spy(new SafeTableClearerKeyValueService( timestampsSupplier::getImmutableTimestamp, new InMemoryKeyValueService(true))); spiedKvs.createTable(TABLE_CONS, metadataBytes(TableMetadataPersistence.SweepStrategy.CONSERVATIVE)); spiedKvs.createTable(TABLE_THOR, metadataBytes(TableMetadataPersistence.SweepStrategy.THOROUGH)); spiedKvs.createTable(TABLE_NOTH, metadataBytes(TableMetadataPersistence.SweepStrategy.NOTHING)); partitioner = new WriteInfoPartitioner(spiedKvs, () -> numShards); txnService = TransactionServices.createV1TransactionService(spiedKvs); }
@Test public void partitionWritesByShardStrategyTimestampPartitionsIntoSeparatePartitions() { List<WriteInfo> writes = ImmutableList.of( getWriteInfo(CONSERVATIVE, 0, 0, 100L), getWriteInfo(CONSERVATIVE, 1, 0, 100L), getWriteInfo(CONSERVATIVE, 0, 3, 100L), getWriteInfo(CONSERVATIVE, 0, 0, 200L), getWriteInfo(CONSERVATIVE2, 0, 0, 100L), getWriteInfo(THOROUGH, 0, 0, 100L)); Map<PartitionInfo, List<WriteInfo>> partitions = partitioner.partitionWritesByShardStrategyTimestamp(writes); assertThat(partitions.size()).isEqualTo(6); }
public void enqueue(List<WriteInfo> allWrites) { Map<Cell, byte[]> referencesToDedicatedCells = new HashMap<>(); Map<Cell, byte[]> cellsToWrite = new HashMap<>(); Map<PartitionInfo, List<WriteInfo>> partitionedWrites = partitioner.filterAndPartition(allWrites); partitionedWrites.forEach((partitionInfo, writes) -> { referencesToDedicatedCells.putAll(populateReferences(partitionInfo, writes)); cellsToWrite.putAll(populateCells(partitionInfo, writes)); }); partitionedWrites.keySet().stream() .map(PartitionInfo::timestamp) .mapToLong(x -> x) .max() .ifPresent(timestamp -> { write(referencesToDedicatedCells, timestamp); write(cellsToWrite, timestamp); updateWriteMetrics(partitionedWrites); }); }
@Test public void getStrategyReturnsCorrectStrategy() { assertThat(partitioner.getStrategy(getWriteInfoWithFixedCellHash(NOTHING, 0))) .isEqualTo(TableMetadataPersistence.SweepStrategy.NOTHING); assertThat(partitioner.getStrategy(getWriteInfoWithFixedCellHash(CONSERVATIVE, 10))) .isEqualTo(TableMetadataPersistence.SweepStrategy.CONSERVATIVE); assertThat(partitioner.getStrategy(getWriteInfoWithFixedCellHash(THOROUGH, 100))) .isEqualTo(TableMetadataPersistence.SweepStrategy.THOROUGH); }
static SweepQueueFactory create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction) { Schemas.createTablesAndIndexes(TargetedSweepSchema.INSTANCE.getLatestSchema(), kvs); ShardProgress shardProgress = new ShardProgress(kvs); Supplier<Integer> shards = createProgressUpdatingSupplier(shardsConfig, shardProgress, SweepQueueUtils.REFRESH_TIME); WriteInfoPartitioner partitioner = new WriteInfoPartitioner(kvs, shards); SweepableCells cells = new SweepableCells(kvs, partitioner, metrics, transaction); SweepableTimestamps timestamps = new SweepableTimestamps(kvs, partitioner); return new SweepQueueFactory(shardProgress, shards, cells, timestamps, metrics, kvs, timelock); }
@Test public void getStrategyThrowsForIllegalMetadata() { when(mockKvs.getMetadataForTable(any())).thenReturn(AtlasDbConstants.EMPTY_TABLE_METADATA); assertThatThrownBy(() -> partitioner.getStrategy(getWriteInfoWithFixedCellHash(getTableRef("a"), 0))) .isInstanceOf(UncheckedExecutionException.class); }