public SweepResults sweepSnapshotIndices() { TodoSchemaTableFactory tableFactory = TodoSchemaTableFactory.of(Namespace.DEFAULT_NAMESPACE); TableReference indexTable = tableFactory.getSnapshotsStreamIdxTable(null).getTableRef(); return sweepTable(indexTable); }
public SweepResults sweepSnapshotValues() { TodoSchemaTableFactory tableFactory = TodoSchemaTableFactory.of(Namespace.DEFAULT_NAMESPACE); TableReference valueTable = tableFactory.getSnapshotsStreamValueTable(null).getTableRef(); return sweepTable(valueTable); }
public static TodoSchemaTableFactory of(List<Function<? super Transaction, SharedTriggers>> sharedTriggers) { return new TodoSchemaTableFactory(sharedTriggers, defaultNamespace); }
public boolean doesNotExistBeforeTimestamp(long id, long ts) { TableReference tableRef = TodoSchemaTableFactory.of().getTodoTable(null).getTableRef(); Cell cell = Cell.create(ValueType.FIXED_LONG.convertFromJava(id), TodoSchema.todoTextColumn()); return kvs.get().get(tableRef, ImmutableMap.of(cell, ts + 1)).isEmpty(); }
smRows.add(SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow.of(streamId)); SnapshotsStreamMetadataTable table = tables.getSnapshotsStreamMetadataTable(t); Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows); Set<SnapshotsStreamValueTable.SnapshotsStreamValueRow> streamValueToDelete = Sets.newHashSet(); shToDelete.put(hashRow, column); tables.getSnapshotsStreamHashAidxTable(t).delete(shToDelete); tables.getSnapshotsStreamValueTable(t).delete(streamValueToDelete); table.delete(smRows);
public SnapshotsMetadataCleanupTask(Namespace namespace) { tables = TodoSchemaTableFactory.of(namespace); }
@Override public void unmarkStreamsAsUsed(Transaction t, final Map<Long, byte[]> streamIdsToReference) { if (streamIdsToReference.isEmpty()) { return; } SnapshotsStreamIdxTable index = tables.getSnapshotsStreamIdxTable(t); Multimap<SnapshotsStreamIdxTable.SnapshotsStreamIdxRow, SnapshotsStreamIdxTable.SnapshotsStreamIdxColumn> toDelete = ArrayListMultimap.create(streamIdsToReference.size(), 1); for (Map.Entry<Long, byte[]> entry : streamIdsToReference.entrySet()) { Long streamId = entry.getKey(); byte[] reference = entry.getValue(); SnapshotsStreamIdxTable.SnapshotsStreamIdxColumn col = SnapshotsStreamIdxTable.SnapshotsStreamIdxColumn.of(reference); toDelete.put(SnapshotsStreamIdxTable.SnapshotsStreamIdxRow.of(streamId), col); } index.delete(toDelete); }
@Override protected Map<Long, StreamMetadata> getMetadata(Transaction t, Set<Long> streamIds) { if (streamIds.isEmpty()) { return ImmutableMap.of(); } SnapshotsStreamMetadataTable table = tables.getSnapshotsStreamMetadataTable(t); Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(getMetadataRowsForIds(streamIds)); Map<Long, StreamMetadata> ret = Maps.newHashMap(); for (Map.Entry<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) { ret.put(e.getKey().getId(), e.getValue()); } return ret; }
private byte[] getBlock(Transaction t, SnapshotsStreamValueTable.SnapshotsStreamValueRow row) { SnapshotsStreamValueTable valueTable = tables.getSnapshotsStreamValueTable(t); return valueTable.getValues(ImmutableSet.of(row)).get(row); }
public long addNamespacedTodoWithIdAndReturnTimestamp(long id, String namespace, Todo todo) { return transactionManager.runTaskWithRetry(tx -> { TodoSchemaTableFactory.of().getNamespacedTodoTable(tx).put( NamespacedTodoTable.NamespacedTodoRow.of(namespace), NamespacedTodoTable.NamespacedTodoColumnValue.of( NamespacedTodoTable.NamespacedTodoColumn.of(id), todo.text())); return tx.getTimestamp(); }); }
public void storeSnapshot(InputStream snapshot) { byte[] streamReference = "EteTest".getBytes(); TodoSchemaTableFactory tableFactory = TodoSchemaTableFactory.of(Namespace.DEFAULT_NAMESPACE); SnapshotsStreamStore streamStore = SnapshotsStreamStore.of(transactionManager, tableFactory); log.info("Storing stream..."); Pair<Long, Sha256Hash> storedStream = streamStore.storeStream(snapshot); Long newStreamId = storedStream.getLhSide(); log.info("Stored stream with ID {}", newStreamId); transactionManager.runTaskWithRetry(transaction -> { // Load previous stream, and unmark it as used LatestSnapshotTable.LatestSnapshotRow row = LatestSnapshotTable.LatestSnapshotRow.of(0L); LatestSnapshotTable latestSnapshotTable = tableFactory.getLatestSnapshotTable(transaction); Optional<LatestSnapshotTable.LatestSnapshotRowResult> maybeRow = latestSnapshotTable.getRow(row); maybeRow.ifPresent(latestSnapshot -> { Long latestStreamId = maybeRow.get().getStreamId(); log.info("Marking stream {}, ref {}, as unused", latestStreamId, PtBytes.toString(streamReference)); Map<Long, byte[]> theMap = ImmutableMap.of(latestStreamId, streamReference); streamStore.unmarkStreamsAsUsed(transaction, theMap); }); streamStore.markStreamAsUsed(transaction, newStreamId, streamReference); log.info("Marked stream {} as used with reference {}", newStreamId, PtBytes.toString(streamReference)); // Record the latest snapshot latestSnapshotTable.putStreamId(row, newStreamId); return null; }); }
@Override public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) { if (hashes.isEmpty()) { return ImmutableMap.of(); } SnapshotsStreamHashAidxTable idx = tables.getSnapshotsStreamHashAidxTable(t); Set<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes); Multimap<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows); Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap(); for (SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow r : m.keySet()) { for (SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue v : m.get(r)) { Long streamId = v.getColumnName().getStreamId(); Sha256Hash hash = r.getHash(); if (hashForStreams.containsKey(streamId)) { AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId); } hashForStreams.put(streamId, hash); } } Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet()); Map<Sha256Hash, Long> ret = Maps.newHashMap(); for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) { if (e.getValue().getStatus() != Status.STORED) { continue; } Sha256Hash hash = hashForStreams.get(e.getKey()); ret.put(hash, e.getKey()); } return ret; }
smRows.add(SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow.of(streamId)); SnapshotsStreamMetadataTable table = tables.getSnapshotsStreamMetadataTable(t); Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> metadatas = table.getMetadatas(smRows); Set<SnapshotsStreamValueTable.SnapshotsStreamValueRow> streamValueToDelete = Sets.newHashSet(); shToDelete.put(hashRow, column); tables.getSnapshotsStreamHashAidxTable(t).delete(shToDelete); tables.getSnapshotsStreamValueTable(t).delete(streamValueToDelete); table.delete(smRows);
public SnapshotsIndexCleanupTask(Namespace namespace) { tables = TodoSchemaTableFactory.of(namespace); }
@Override protected void markStreamsAsUsedInternal(Transaction t, final Map<Long, byte[]> streamIdsToReference) { if (streamIdsToReference.isEmpty()) { return; } SnapshotsStreamIdxTable index = tables.getSnapshotsStreamIdxTable(t); Multimap<SnapshotsStreamIdxTable.SnapshotsStreamIdxRow, SnapshotsStreamIdxTable.SnapshotsStreamIdxColumnValue> rowsToValues = HashMultimap.create(); for (Map.Entry<Long, byte[]> entry : streamIdsToReference.entrySet()) { Long streamId = entry.getKey(); byte[] reference = entry.getValue(); SnapshotsStreamIdxTable.SnapshotsStreamIdxColumn col = SnapshotsStreamIdxTable.SnapshotsStreamIdxColumn.of(reference); SnapshotsStreamIdxTable.SnapshotsStreamIdxColumnValue value = SnapshotsStreamIdxTable.SnapshotsStreamIdxColumnValue.of(col, 0L); rowsToValues.put(SnapshotsStreamIdxTable.SnapshotsStreamIdxRow.of(streamId), value); } index.put(rowsToValues); }
@Override protected void touchMetadataWhileMarkingUsedForConflicts(Transaction t, Iterable<Long> ids) { SnapshotsStreamMetadataTable metaTable = tables.getSnapshotsStreamMetadataTable(t); Set<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow> rows = Sets.newHashSet(); for (Long id : ids) { rows.add(SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow.of(id)); } Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> metadatas = metaTable.getMetadatas(rows); for (Map.Entry<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> e : metadatas.entrySet()) { StreamMetadata metadata = e.getValue(); Preconditions.checkState(metadata.getStatus() == Status.STORED, "Stream: %s has status: %s", e.getKey().getId(), metadata.getStatus()); metaTable.putMetadata(e.getKey(), metadata); } SetView<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow> missingRows = Sets.difference(rows, metadatas.keySet()); if (!missingRows.isEmpty()) { throw new IllegalStateException("Missing metadata rows for:" + missingRows + " rows: " + rows + " metadata: " + metadatas + " txn timestamp: " + t.getTimestamp()); } }
@Override protected void storeBlock(Transaction t, long id, long blockNumber, final byte[] block) { Preconditions.checkArgument(block.length <= BLOCK_SIZE_IN_BYTES, "Block to store in DB must be less than BLOCK_SIZE_IN_BYTES"); final SnapshotsStreamValueTable.SnapshotsStreamValueRow row = SnapshotsStreamValueTable.SnapshotsStreamValueRow.of(id, blockNumber); try { // Do a touch operation on this table to ensure we get a conflict if someone cleans it up. touchMetadataWhileStoringForConflicts(t, row.getId(), row.getBlockId()); tables.getSnapshotsStreamValueTable(t).putValue(row, block); } catch (RuntimeException e) { log.error("Error storing block {} for stream id {}", row.getBlockId(), row.getId(), e); throw e; } }
private void putHashIndexTask(Transaction t, Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> rowsToMetadata) { Multimap<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue> indexMap = HashMultimap.create(); for (Entry<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> e : rowsToMetadata.entrySet()) { SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow row = e.getKey(); StreamMetadata metadata = e.getValue(); Preconditions.checkArgument( metadata.getStatus() == Status.STORED, "Should only index successfully stored streams."); Sha256Hash hash = Sha256Hash.EMPTY; if (metadata.getHash() != com.google.protobuf.ByteString.EMPTY) { hash = new Sha256Hash(metadata.getHash().toByteArray()); } SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow hashRow = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow.of(hash); SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumn column = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumn.of(row.getId()); SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue columnValue = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue.of(column, 0L); indexMap.put(hashRow, columnValue); } SnapshotsStreamHashAidxTable hiTable = tables.getSnapshotsStreamHashAidxTable(t); hiTable.put(indexMap); }
public static TodoSchemaTableFactory of() { return of(ImmutableList.<Function<? super Transaction, SharedTriggers>>of(), defaultNamespace); }
@Override public boolean cellsCleanedUp(Transaction t, Set<Cell> cells) { SnapshotsStreamIdxTable usersIndex = tables.getSnapshotsStreamIdxTable(t); Set<SnapshotsStreamIdxTable.SnapshotsStreamIdxRow> rows = Sets.newHashSetWithExpectedSize(cells.size()); for (Cell cell : cells) { rows.add(SnapshotsStreamIdxTable.SnapshotsStreamIdxRow.BYTES_HYDRATOR.hydrateFromBytes(cell.getRowName())); } BatchColumnRangeSelection oneColumn = BatchColumnRangeSelection.create( PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY, 1); Map<SnapshotsStreamIdxTable.SnapshotsStreamIdxRow, BatchingVisitable<SnapshotsStreamIdxTable.SnapshotsStreamIdxColumnValue>> existentRows = usersIndex.getRowsColumnRange(rows, oneColumn); Set<SnapshotsStreamIdxTable.SnapshotsStreamIdxRow> rowsInDb = Sets.newHashSetWithExpectedSize(cells.size()); for (Map.Entry<SnapshotsStreamIdxTable.SnapshotsStreamIdxRow, BatchingVisitable<SnapshotsStreamIdxTable.SnapshotsStreamIdxColumnValue>> rowVisitable : existentRows.entrySet()) { rowVisitable.getValue().batchAccept(1, columnValues -> { if (!columnValues.isEmpty()) { rowsInDb.add(rowVisitable.getKey()); } return false; }); } Set<Long> toDelete = Sets.newHashSetWithExpectedSize(rows.size() - rowsInDb.size()); for (SnapshotsStreamIdxTable.SnapshotsStreamIdxRow rowToDelete : Sets.difference(rows, rowsInDb)) { toDelete.add(rowToDelete.getId()); } SnapshotsStreamStore.of(tables).deleteStreams(t, toDelete); return false; } }