@Override public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) { if (hashes.isEmpty()) { return ImmutableMap.of(); } SnapshotsStreamHashAidxTable idx = tables.getSnapshotsStreamHashAidxTable(t); Set<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes); Multimap<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows); Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap(); for (SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow r : m.keySet()) { for (SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue v : m.get(r)) { Long streamId = v.getColumnName().getStreamId(); Sha256Hash hash = r.getHash(); if (hashForStreams.containsKey(streamId)) { AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId); } hashForStreams.put(streamId, hash); } } Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet()); Map<Sha256Hash, Long> ret = Maps.newHashMap(); for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) { if (e.getValue().getStatus() != Status.STORED) { continue; } Sha256Hash hash = hashForStreams.get(e.getKey()); ret.put(hash, e.getKey()); } return ret; }
@Override public List<SnapshotsStreamHashAidxColumnValue> getRowColumns(SnapshotsStreamHashAidxRow row, ColumnSelection columns) { byte[] bytes = row.persistToBytes(); RowResult<byte[]> rowResult = t.getRows(tableRef, ImmutableSet.of(bytes), columns).get(bytes); if (rowResult == null) { return ImmutableList.of(); } else { List<SnapshotsStreamHashAidxColumnValue> ret = Lists.newArrayListWithCapacity(rowResult.getColumns().size()); for (Entry<byte[], byte[]> e : rowResult.getColumns().entrySet()) { SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); ret.add(SnapshotsStreamHashAidxColumnValue.of(col, val)); } return ret; } }
@Override public Map<Sha256Hash, Long> lookupStreamIdsByHash(Transaction t, final Set<Sha256Hash> hashes) { if (hashes.isEmpty()) { return ImmutableMap.of(); } SnapshotsStreamHashAidxTable idx = tables.getSnapshotsStreamHashAidxTable(t); Set<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow> rows = getHashIndexRowsForHashes(hashes); Multimap<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue> m = idx.getRowsMultimap(rows); Map<Long, Sha256Hash> hashForStreams = Maps.newHashMap(); for (SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow r : m.keySet()) { for (SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue v : m.get(r)) { Long streamId = v.getColumnName().getStreamId(); Sha256Hash hash = r.getHash(); if (hashForStreams.containsKey(streamId)) { AssertUtils.assertAndLog(log, hashForStreams.get(streamId).equals(hash), "(BUG) Stream ID has 2 different hashes: " + streamId); } hashForStreams.put(streamId, hash); } } Map<Long, StreamMetadata> metadata = getMetadata(t, hashForStreams.keySet()); Map<Sha256Hash, Long> ret = Maps.newHashMap(); for (Map.Entry<Long, StreamMetadata> e : metadata.entrySet()) { if (e.getValue().getStatus() != Status.STORED) { continue; } Sha256Hash hash = hashForStreams.get(e.getKey()); ret.put(hash, e.getKey()); } return ret; }
private void putHashIndexTask(Transaction t, Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> rowsToMetadata) { Multimap<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue> indexMap = HashMultimap.create(); for (Entry<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> e : rowsToMetadata.entrySet()) { SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow row = e.getKey(); StreamMetadata metadata = e.getValue(); Preconditions.checkArgument( metadata.getStatus() == Status.STORED, "Should only index successfully stored streams."); Sha256Hash hash = Sha256Hash.EMPTY; if (metadata.getHash() != com.google.protobuf.ByteString.EMPTY) { hash = new Sha256Hash(metadata.getHash().toByteArray()); } SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow hashRow = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow.of(hash); SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumn column = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumn.of(row.getId()); SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue columnValue = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue.of(column, 0L); indexMap.put(hashRow, columnValue); } SnapshotsStreamHashAidxTable hiTable = tables.getSnapshotsStreamHashAidxTable(t); hiTable.put(indexMap); }
@Override public List<SnapshotsStreamHashAidxColumnValue> getRowColumns(SnapshotsStreamHashAidxRow row, ColumnSelection columns) { byte[] bytes = row.persistToBytes(); RowResult<byte[]> rowResult = t.getRows(tableRef, ImmutableSet.of(bytes), columns).get(bytes); if (rowResult == null) { return ImmutableList.of(); } else { List<SnapshotsStreamHashAidxColumnValue> ret = Lists.newArrayListWithCapacity(rowResult.getColumns().size()); for (Entry<byte[], byte[]> e : rowResult.getColumns().entrySet()) { SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); ret.add(SnapshotsStreamHashAidxColumnValue.of(col, val)); } return ret; } }
@Override public Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> get(Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumn> cells) { Set<Cell> rawCells = ColumnValues.toCells(cells); Map<Cell, byte[]> rawResults = t.get(tableRef, rawCells); Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> rowMap = HashMultimap.create(); for (Entry<Cell, byte[]> e : rawResults.entrySet()) { if (e.getValue().length > 0) { SnapshotsStreamHashAidxRow row = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(e.getKey().getRowName()); SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey().getColumnName()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); rowMap.put(row, SnapshotsStreamHashAidxColumnValue.of(col, val)); } } return rowMap; }
@Override public Map<SnapshotsStreamHashAidxRow, BatchingVisitable<SnapshotsStreamHashAidxColumnValue>> getRowsColumnRange(Iterable<SnapshotsStreamHashAidxRow> rows, BatchColumnRangeSelection columnRangeSelection) { Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> results = t.getRowsColumnRange(tableRef, Persistables.persistAll(rows), columnRangeSelection); Map<SnapshotsStreamHashAidxRow, BatchingVisitable<SnapshotsStreamHashAidxColumnValue>> transformed = Maps.newHashMapWithExpectedSize(results.size()); for (Entry<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> e : results.entrySet()) { SnapshotsStreamHashAidxRow row = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(e.getKey()); BatchingVisitable<SnapshotsStreamHashAidxColumnValue> bv = BatchingVisitables.transform(e.getValue(), result -> { SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(result.getKey().getColumnName()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(result.getValue()); return SnapshotsStreamHashAidxColumnValue.of(col, val); }); transformed.put(row, bv); } return transformed; }
@Override public Long apply(SnapshotsStreamHashAidxColumnValue columnValue) { return columnValue.getValue(); } };
public static SnapshotsStreamHashAidxColumnValue of(SnapshotsStreamHashAidxColumn columnName, Long value) { return new SnapshotsStreamHashAidxColumnValue(columnName, value); }
@Override public Iterator<Map.Entry<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue>> getRowsColumnRange(Iterable<SnapshotsStreamHashAidxRow> rows, ColumnRangeSelection columnRangeSelection, int batchHint) { Iterator<Map.Entry<Cell, byte[]>> results = t.getRowsColumnRange(getTableRef(), Persistables.persistAll(rows), columnRangeSelection, batchHint); return Iterators.transform(results, e -> { SnapshotsStreamHashAidxRow row = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(e.getKey().getRowName()); SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey().getColumnName()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); SnapshotsStreamHashAidxColumnValue colValue = SnapshotsStreamHashAidxColumnValue.of(col, val); return Maps.immutableEntry(row, colValue); }); }
@Override public void touch(Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumn> values) { Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> currentValues = get(values); put(currentValues); Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumn> toDelete = HashMultimap.create(values); for (Map.Entry<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> e : currentValues.entries()) { toDelete.remove(e.getKey(), e.getValue().getColumnName()); } delete(toDelete); }
@Override public SnapshotsStreamHashAidxColumn apply(SnapshotsStreamHashAidxColumnValue columnValue) { return columnValue.getColumnName(); } };
public static SnapshotsStreamHashAidxRowResult of(RowResult<byte[]> rowResult) { SnapshotsStreamHashAidxRow rowName = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(rowResult.getRowName()); Set<SnapshotsStreamHashAidxColumnValue> columnValues = Sets.newHashSetWithExpectedSize(rowResult.getColumns().size()); for (Entry<byte[], byte[]> e : rowResult.getColumns().entrySet()) { SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey()); Long value = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); columnValues.add(SnapshotsStreamHashAidxColumnValue.of(col, value)); } return new SnapshotsStreamHashAidxRowResult(rowName, ImmutableSet.copyOf(columnValues)); }
@Override public void delete(Iterable<SnapshotsStreamHashAidxRow> rows) { Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumn> toRemove = HashMultimap.create(); Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> result = getRowsMultimap(rows); for (Entry<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> e : result.entries()) { toRemove.put(e.getKey(), e.getValue().getColumnName()); } delete(toRemove); }
private static Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> getRowMapFromRowResults(Collection<RowResult<byte[]>> rowResults) { Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> rowMap = HashMultimap.create(); for (RowResult<byte[]> result : rowResults) { SnapshotsStreamHashAidxRow row = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(result.getRowName()); for (Entry<byte[], byte[]> e : result.getColumns().entrySet()) { SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); rowMap.put(row, SnapshotsStreamHashAidxColumnValue.of(col, val)); } } return rowMap; }
private void putHashIndexTask(Transaction t, Map<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> rowsToMetadata) { Multimap<SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue> indexMap = HashMultimap.create(); for (Entry<SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow, StreamMetadata> e : rowsToMetadata.entrySet()) { SnapshotsStreamMetadataTable.SnapshotsStreamMetadataRow row = e.getKey(); StreamMetadata metadata = e.getValue(); Preconditions.checkArgument( metadata.getStatus() == Status.STORED, "Should only index successfully stored streams."); Sha256Hash hash = Sha256Hash.EMPTY; if (metadata.getHash() != com.google.protobuf.ByteString.EMPTY) { hash = new Sha256Hash(metadata.getHash().toByteArray()); } SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow hashRow = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxRow.of(hash); SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumn column = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumn.of(row.getId()); SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue columnValue = SnapshotsStreamHashAidxTable.SnapshotsStreamHashAidxColumnValue.of(column, 0L); indexMap.put(hashRow, columnValue); } SnapshotsStreamHashAidxTable hiTable = tables.getSnapshotsStreamHashAidxTable(t); hiTable.put(indexMap); }
@Override public Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> get(Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumn> cells) { Set<Cell> rawCells = ColumnValues.toCells(cells); Map<Cell, byte[]> rawResults = t.get(tableRef, rawCells); Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> rowMap = HashMultimap.create(); for (Entry<Cell, byte[]> e : rawResults.entrySet()) { if (e.getValue().length > 0) { SnapshotsStreamHashAidxRow row = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(e.getKey().getRowName()); SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(e.getKey().getColumnName()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(e.getValue()); rowMap.put(row, SnapshotsStreamHashAidxColumnValue.of(col, val)); } } return rowMap; }
public static SnapshotsStreamHashAidxColumnValue of(SnapshotsStreamHashAidxColumn columnName, Long value) { return new SnapshotsStreamHashAidxColumnValue(columnName, value); }
@Override public Map<SnapshotsStreamHashAidxRow, BatchingVisitable<SnapshotsStreamHashAidxColumnValue>> getRowsColumnRange(Iterable<SnapshotsStreamHashAidxRow> rows, BatchColumnRangeSelection columnRangeSelection) { Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> results = t.getRowsColumnRange(tableRef, Persistables.persistAll(rows), columnRangeSelection); Map<SnapshotsStreamHashAidxRow, BatchingVisitable<SnapshotsStreamHashAidxColumnValue>> transformed = Maps.newHashMapWithExpectedSize(results.size()); for (Entry<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> e : results.entrySet()) { SnapshotsStreamHashAidxRow row = SnapshotsStreamHashAidxRow.BYTES_HYDRATOR.hydrateFromBytes(e.getKey()); BatchingVisitable<SnapshotsStreamHashAidxColumnValue> bv = BatchingVisitables.transform(e.getValue(), result -> { SnapshotsStreamHashAidxColumn col = SnapshotsStreamHashAidxColumn.BYTES_HYDRATOR.hydrateFromBytes(result.getKey().getColumnName()); Long val = SnapshotsStreamHashAidxColumnValue.hydrateValue(result.getValue()); return SnapshotsStreamHashAidxColumnValue.of(col, val); }); transformed.put(row, bv); } return transformed; }
@Override public void delete(Iterable<SnapshotsStreamHashAidxRow> rows) { Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumn> toRemove = HashMultimap.create(); Multimap<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> result = getRowsMultimap(rows); for (Entry<SnapshotsStreamHashAidxRow, SnapshotsStreamHashAidxColumnValue> e : result.entries()) { toRemove.put(e.getKey(), e.getValue().getColumnName()); } delete(toRemove); }