/** * This is a replacement for startRow when doing reverse range request. */ public static byte[] startRowInclusiveOrLargestRow(RangeRequest rangeRequest) { Preconditions.checkArgument(rangeRequest.isReverse()); if (rangeRequest.getStartInclusive().length == 0) { return getLastRowName(); } return rangeRequest.getStartInclusive(); }
@Override protected TokenBackedBasicResultsPage<RowResult<T>, byte[]> getFirstPage() throws Exception { return getSinglePage(rangeRequest.getStartInclusive()); }
public static WhereClauses create(String tableIdentifier, RangeRequest request, String... clauses) { List<String> extraWhereClauses = Lists.newArrayList(clauses); byte[] start = request.getStartInclusive(); byte[] end = request.getEndExclusive(); Collection<byte[]> cols = request.getColumnNames(); List<Object> args = Lists.newArrayListWithCapacity(2 + cols.size()); List<String> whereClauses = Lists.newArrayListWithCapacity(3 + extraWhereClauses.size()); if (start.length > 0) { whereClauses.add(tableIdentifier + (request.isReverse() ? ".row_name <= ?" : ".row_name >= ?")); args.add(start); } if (end.length > 0) { whereClauses.add(tableIdentifier + (request.isReverse() ? ".row_name > ?" : ".row_name < ?")); args.add(end); } if (!cols.isEmpty()) { whereClauses.add(tableIdentifier + ".col_name IN (" + BasicSQLUtils.nArguments(cols.size()) + ")"); args.addAll(cols); } whereClauses.addAll(extraWhereClauses); return new WhereClauses(whereClauses, args); }
public boolean inRange(byte[] position) { Preconditions.checkArgument(Cell.isNameValid(position)); final boolean afterStart; final boolean afterEnd; if (reverse) { afterStart = getStartInclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getStartInclusive(), position) >= 0; afterEnd = getEndExclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getEndExclusive(), position) < 0; } else { afterStart = getStartInclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getStartInclusive(), position) <= 0; afterEnd = getEndExclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getEndExclusive(), position) > 0; } return afterStart && afterEnd; }
@Override public FullQuery getRangeQuery(RangeRequest range, long ts, int maxRows) { List<String> bounds = Lists.newArrayListWithCapacity(2); List<Object> args = Lists.newArrayListWithCapacity(2); byte[] start = range.getStartInclusive(); byte[] end = range.getEndExclusive(); if (start.length > 0) { bounds.add(range.isReverse() ? "m.row_name <= ?" : "m.row_name >= ?"); args.add(start); } if (end.length > 0) { bounds.add(range.isReverse() ? "m.row_name > ?" : "m.row_name < ?"); args.add(end); } String query = " /* GET_RANGE_ROWS (" + tableName + ") */ " + " SELECT DISTINCT m.row_name " + " FROM " + prefixedTableName() + " m " + (bounds.isEmpty() ? "" : " WHERE " + Joiner.on(" AND ").join(bounds)) + " ORDER BY m.row_name " + (range.isReverse() ? "DESC" : "ASC") + " LIMIT " + maxRows; return new FullQuery(query).withArgs(args); }
private List<Entry<Cell, byte[]>> getPostFilteredWithLocalWrites(final TableReference tableRef, final SortedMap<Cell, byte[]> postFiltered, final RangeRequest rangeRequest, List<RowResult<Value>> prePostFilter, final byte[] endRowExclusive) { Map<Cell, Value> prePostFilterCells = Cells.convertRowResultsToCells(prePostFilter); Collection<Entry<Cell, byte[]>> postFilteredCells = Collections2.filter( postFiltered.entrySet(), Predicates.compose( Predicates.in(prePostFilterCells.keySet()), MapEntries.getKeyFunction())); Collection<Entry<Cell, byte[]>> localWritesInRange = getLocalWritesForRange( tableRef, rangeRequest.getStartInclusive(), endRowExclusive).entrySet(); return mergeInLocalWrites( tableRef, postFilteredCells.iterator(), localWritesInRange.iterator(), rangeRequest.isReverse()); }
private Object getMultiRangeInner(ConsecutiveNarrowTable table) { Iterable<RangeRequest> requests = table.getRangeRequests(1000, 1, false); Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> results = table.getKvs().getFirstBatchForRanges(table.getTableRef(), requests, Long.MAX_VALUE); int numRequests = Iterables.size(requests); Preconditions.checkState(numRequests == results.size(), "Got %s requests and %s results, requests %s, results %s", numRequests, results.size(), requests, results); results.forEach((request, result) -> { Preconditions.checkState(1 == result.getResults().size(), "Key %s, List size is %s", Ints.fromByteArray(request.getStartInclusive()), result.getResults().size()); Preconditions.checkState(!result.moreResultsAvailable(), "Key %s, result.moreResultsAvailable() %s", Ints.fromByteArray(request.getStartInclusive()), (Object) result.moreResultsAvailable()); RowResult<Value> row = Iterables.getOnlyElement(result.getResults()); Preconditions.checkState(Arrays.equals(request.getStartInclusive(), row.getRowName()), "Request row is %s, result is %s", Ints.fromByteArray(request.getStartInclusive()), Ints.fromByteArray(row.getRowName())); }); return results; }
private Object getSingleRangeInner(ConsecutiveNarrowTable table, int sliceSize) { RangeRequest request = Iterables.getOnlyElement(table.getRangeRequests(1, sliceSize, false)); int startRow = Ints.fromByteArray(request.getStartInclusive()); ClosableIterator<RowResult<Value>> result = table.getKvs().getRange(table.getTableRef(), request, Long.MAX_VALUE); ArrayList<RowResult<Value>> list = Lists.newArrayList(result); result.close(); Preconditions.checkState(list.size() == sliceSize, "List size %s != %s", sliceSize, list.size()); list.forEach(rowResult -> { byte[] rowName = rowResult.getRowName(); int rowNumber = Ints.fromByteArray(rowName); Preconditions.checkState(rowNumber - startRow < sliceSize, "Start Row %s, row number %s, sliceSize %s", startRow, rowNumber, sliceSize); }); return result; }
private Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> getFirstPages( TableReference tableRef, List<RangeRequest> requests, long timestamp) { List<String> subQueries = Lists.newArrayList(); List<Object> argsList = Lists.newArrayList(); for (int i = 0; i < requests.size(); i++) { RangeRequest request = requests.get(i); Pair<String, List<Object>> queryAndArgs = getRangeQueryAndArgs( tableRef, request.getStartInclusive(), request.getEndExclusive(), request.isReverse(), request.getBatchHint() == null ? 1 : request.getBatchHint(), i); subQueries.add(queryAndArgs.lhSide); argsList.addAll(queryAndArgs.rhSide); } String query = Joiner.on(") UNION ALL (").appendTo(new StringBuilder("("), subQueries).append(")").toString(); Object[] args = argsList.toArray(); TimingState timer = logTimer.begin("Table: " + tableRef.getQualifiedName() + " get_page"); try { return getFirstPagesFromDb(tableRef, requests, timestamp, query, args); } finally { timer.end(); } }
private NavigableMap<Cell, byte[]> getReadsInRange(TableReference table, RangeRequest range) { NavigableMap<Cell, byte[]> reads = getReadsForTable(table); if (range.getStartInclusive().length != 0) { reads = reads.tailMap(Cells.createSmallestCellForRow(range.getStartInclusive()), true); } if (range.getEndExclusive().length != 0) { reads = reads.headMap(Cells.createSmallestCellForRow(range.getEndExclusive()), false); } Map<Cell, byte[]> writes = writesByTable.get(table); if (writes != null) { reads = Maps.filterKeys(reads, Predicates.not(Predicates.in(writes.keySet()))); } if (!range.getColumnNames().isEmpty()) { Predicate<Cell> columnInNames = Predicates.compose( Predicates.in(range.getColumnNames()), Cell::getColumnName); reads = Maps.filterKeys(reads, columnInNames); } return reads; }
List<String> bounds = Lists.newArrayListWithCapacity(2); List<Object> args = Lists.newArrayListWithCapacity(2); byte[] start = range.getStartInclusive(); byte[] end = range.getEndExclusive(); if (start.length > 0) {
private List<RowResult<byte[]>> getSingleRowWithRangeQueryInner(final ConsecutiveNarrowTable table) { return table.getTransactionManager().runTaskThrowOnConflict(txn -> { RangeRequest request = Iterables.getOnlyElement(table.getRangeRequests(1, 1, false)); List<RowResult<byte[]>> result = BatchingVisitables.copyToList( txn.getRange(table.getTableRef(), request)); byte[] rowName = Iterables.getOnlyElement(result).getRowName(); int rowNumber = ConsecutiveNarrowTable.rowNumber(rowName); int expectedRowNumber = ConsecutiveNarrowTable.rowNumber(request.getStartInclusive()); Preconditions.checkState(rowNumber == expectedRowNumber, "Start Row %s, row number %s", expectedRowNumber, rowNumber); return result; }); }
@Override public Iterator<RowResult<Value>> getRange(TableReference tableRef, RangeRequest rangeRequest, long timestamp) { boolean haveOverflow = checkIfTableHasOverflowUsingNewConnection(tableRef); return Iterators.concat(new PageIterator( rangeRequest.getStartInclusive(), rangeRequest.getEndExclusive(), rangeRequest.getColumnNames(), rangeRequest.isReverse(), tableRef, haveOverflow, RangeHelpers.getMaxRowsPerPage(rangeRequest), timestamp)); }
private byte[] copyOneTransactionInternal(RangeRequest range, long rangeId, Transaction readT, Transaction writeT) { final long maxBytes = TransactionConstants.WARN_LEVEL_FOR_QUEUED_BYTES / 2; byte[] start = getCheckpoint(rangeId, writeT); if (start == null) { return null; } RangeRequest.Builder builder = range.getBuilder().startRowInclusive(start); if (builder.isInvalidRange()) { return null; } RangeRequest rangeToUse = builder.build(); if (log.isTraceEnabled()) { log.trace("Copying table {} range {} from {} to {}", srcTable, rangeId, BaseEncoding.base16().lowerCase().encode(rangeToUse.getStartInclusive()), BaseEncoding.base16().lowerCase().encode(rangeToUse.getEndExclusive())); } BatchingVisitable<RowResult<byte[]>> bv = readT.getRange(srcTable, rangeToUse); Map<Cell, byte[]> writeMap = Maps.newHashMap(); byte[] lastRow = internalCopyRange(bv, maxBytes, writeMap); if (log.isTraceEnabled() && (lastRow != null)) { log.trace("Copying {} bytes for range {} on table {}", lastRow.length, rangeId, srcTable); } writeToKvs(writeMap); byte[] nextRow = getNextRowName(lastRow); checkpointer.checkpoint(srcTable.getQualifiedName(), rangeId, nextRow, writeT); return lastRow; }
request.getStartInclusive(), request.getEndExclusive(), request.isReverse());
@Override public Iterator<RowResult<Value>> getRange(TableReference tableRef, RangeRequest rangeRequest, long timestamp) { int maxRowsPerPage = RangeHelpers.getMaxRowsPerPage(rangeRequest); int cellsPerRowEstimate = getCellsPerRowEstimate(tableRef, rangeRequest); int maxCellsPerPage = Math.min( AtlasDbPerformanceConstants.MAX_BATCH_SIZE, maxRowsPerPage * cellsPerRowEstimate) + 1; String tableName = DbKvs.internalTableName(tableRef); Iterator<Iterator<RowResult<Value>>> pageIterator = new PageIterator( rangeRequest.getStartInclusive(), rangeRequest.getEndExclusive(), rangeRequest.getColumnNames(), rangeRequest.isReverse(), timestamp, maxRowsPerPage, maxCellsPerPage, tableName, prefixedTableNames.get(tableRef)); return Iterators.concat(pageIterator); }
@Override public void deleteRange(final TableReference tableRef, final RangeRequest range) { if (range.equals(RangeRequest.all())) { try { cassandraTableTruncator.truncateTables(ImmutableSet.of(tableRef)); } catch (AtlasDbDependencyException e) { log.info("Tried to make a deleteRange({}, RangeRequest.all())" + " into a more garbage-cleanup friendly truncate(), but this failed.", LoggingArgs.tableRef(tableRef), e); super.deleteRange(tableRef, range); } } else if (isForSingleRow(range.getStartInclusive(), range.getEndExclusive())) { try { long timestamp = mutationTimestampProvider.getRemoveTimestamp(); byte[] row = range.getStartInclusive(); clientPool.runWithRetry(client -> { client.remove("deleteRange", tableRef, row, timestamp, DELETE_CONSISTENCY); return null; }); } catch (UnavailableException e) { throw new InsufficientConsistencyException( "Deleting requires all Cassandra nodes to be up and available.", e); } catch (TException e) { throw Throwables.unwrapAndThrowAtlasDbDependencyException(e); } } else { super.deleteRange(tableRef, range); } }
private <K extends Exception> boolean getBatchingVisitableFromIterator( TableReference tableRef, RangeRequest range, int userRequestedSize, AbortingVisitor<List<RowResult<byte[]>>, K> visitor, int preFilterBatchSize) throws K { ClosableIterator<RowResult<byte[]>> postFilterIterator = postFilterIterator(tableRef, range, preFilterBatchSize, Value.GET_VALUE); try { Iterator<RowResult<byte[]>> localWritesInRange = Cells.createRowView( getLocalWritesForRange(tableRef, range.getStartInclusive(), range.getEndExclusive()).entrySet()); Iterator<RowResult<byte[]>> mergeIterators = mergeInLocalWritesRows(postFilterIterator, localWritesInRange, range.isReverse()); return BatchingVisitableFromIterable.create(mergeIterators).batchAccept(userRequestedSize, visitor); } finally { postFilterIterator.close(); } }
long timestamp) { CandidateCellForSweepingRequest request = ImmutableCandidateCellForSweepingRequest.builder() .startRowInclusive(rangeRequest.getStartInclusive()) .maxTimestampExclusive(timestamp) .shouldCheckIfLatestValueIsEmpty(false)
private SelectOffsetStep<Record1<byte[]>> getRangeQuery(DSLContext ctx, TableReference tableRef, RangeRequest rangeRequest, long timestamp, int maxRows) { boolean reverse = rangeRequest.isReverse(); byte[] start = rangeRequest.getStartInclusive(); byte[] end = rangeRequest.getEndExclusive(); Condition cond = R_TIMESTAMP.lessThan(timestamp); if (start.length > 0) { cond = cond.and(reverse ? R_ROW_NAME.lessOrEqual(start) : R_ROW_NAME.greaterOrEqual(start)); } if (end.length > 0) { cond = cond.and(reverse ? R_ROW_NAME.greaterThan(end) : R_ROW_NAME.lessThan(end)); } return ctx.selectDistinct(R_ROW_NAME) .from(atlasTable(tableRef).as(RANGE_TABLE)) .where(cond) .orderBy(reverse ? R_ROW_NAME.desc() : R_ROW_NAME.asc()) .limit(maxRows); }