private void _switchNow() throws IOException { if (onlySwitchAfterRow) throw new IllegalStateException("Can only switch on row boundries"); if (switchSource()) { if (key != null) { iter.seek(new Range(key, true, range.getEndKey(), range.isEndKeyInclusive()), columnFamilies, inclusive); } } }
static void trackScanning(Map<KeyExtent,List<Range>> failures, Map<KeyExtent,List<Range>> unscanned, MultiScanResult scanResult) { // translate returned failures, remove them from unscanned, and add them to failures Map<KeyExtent,List<Range>> retFailures = Translator.translate(scanResult.failures, Translators.TKET, new Translator.ListTranslator<>(Translators.TRT)); unscanned.keySet().removeAll(retFailures.keySet()); failures.putAll(retFailures); // translate full scans and remove them from unscanned HashSet<KeyExtent> fullScans = new HashSet<>( Translator.translate(scanResult.fullScans, Translators.TKET)); unscanned.keySet().removeAll(fullScans); // remove partial scan from unscanned if (scanResult.partScan != null) { KeyExtent ke = new KeyExtent(scanResult.partScan); Key nextKey = new Key(scanResult.partNextKey); ListIterator<Range> iterator = unscanned.get(ke).listIterator(); while (iterator.hasNext()) { Range range = iterator.next(); if (range.afterEndKey(nextKey) || (nextKey.equals(range.getEndKey()) && scanResult.partNextKeyInclusive != range.isEndKeyInclusive())) { iterator.remove(); } else if (range.contains(nextKey)) { iterator.remove(); Range partRange = new Range(nextKey, scanResult.partNextKeyInclusive, range.getEndKey(), range.isEndKeyInclusive()); iterator.add(partRange); } } } }
private void reseek(Key key) throws IOException { if (range.afterEndKey(key)) { range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive()); source.seek(range, columnFamilies, inclusive); } else { range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive()); source.seek(range, columnFamilies, inclusive); } }
private void reseek(Key key) throws IOException { if (range.afterEndKey(key)) { range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive()); source.seek(range, colFamSet, inclusive); } else { range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive()); source.seek(range, colFamSet, inclusive); } }
@Override public boolean getCandidates(String continuePoint, List<String> result) throws TableNotFoundException { // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing // them, // then will never inspect deletes after N Range range = MetadataSchema.DeletesSection.getRange(); if (continuePoint != null && !continuePoint.isEmpty()) { String continueRow = MetadataSchema.DeletesSection.getRowPrefix() + continuePoint; range = new Range(new Key(continueRow).followingKey(PartialKey.ROW), true, range.getEndKey(), range.isEndKeyInclusive()); } Scanner scanner = getClient().createScanner(tableName, Authorizations.EMPTY); scanner.setRange(range); result.clear(); // find candidates for deletion; chop off the prefix for (Entry<Key,Value> entry : scanner) { String cand = entry.getKey().getRow().toString() .substring(MetadataSchema.DeletesSection.getRowPrefix().length()); result.add(cand); if (almostOutOfMemory(Runtime.getRuntime())) { log.info("List of delete candidates has exceeded the memory" + " threshold. Attempting to delete what has been gathered so far."); return true; } } return false; }
private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) { if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) { Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive()); lookupResult.unfinishedRanges.add(nlur); } }
protected void reseek(Key key) throws IOException { if (key == null) return; if (range.afterEndKey(key)) { range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive()); getSource().seek(range, columnFamilies, inclusive); } else { range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive()); getSource().seek(range, columnFamilies, inclusive); } }
private void resetSource() { if (prevTablet == null) { source = iteratorFactory.apply(range); } else { // get the metadata table row for the previous tablet Text prevMetaRow = TabletsSection.getRow(prevTablet.getTableId(), prevTablet.getEndRow()); // ensure the previous tablet still exists in the metadata table if (Iterators.size(iteratorFactory.apply(new Range(prevMetaRow))) == 0) { throw new TabletDeletedException("Tablet " + prevMetaRow + " was deleted while iterating"); } // start scanning at next possible row in metadata table Range seekRange = new Range(new Key(prevMetaRow).followingKey(PartialKey.ROW), true, range.getEndKey(), range.isEndKeyInclusive()); log.info("Resetting scanner to {}", seekRange); source = iteratorFactory.apply(seekRange); } }
TreeMap<Key,Value> consumeMany(Collection<SortedKeyValueIterator<Key,Value>> iterators, Range range, Collection<ByteSequence> seekColumnFamilies, boolean seekInclusive) throws IOException { TreeMap<Key,Value> data = new TreeMap<>(); // All of the copies should have consistent results from concurrent use while (allHasTop(iterators)) { // occasionally deep copy one of the existing iterators if (random.nextInt(3) == 0) { log.debug("Deep-copying and re-seeking an iterator"); SortedKeyValueIterator<Key,Value> newcopy = getRandomElement(iterators) .deepCopy(new SimpleIteratorEnvironment()); newcopy.seek( new Range(getTopKey(iterators), true, range.getEndKey(), range.isEndKeyInclusive()), seekColumnFamilies, seekInclusive); // keep using the new one too, should act like the others iterators.add(newcopy); } data.put(getTopKey(iterators), getTopValue(iterators)); next(iterators); } // All of the iterators should be consumed. for (SortedKeyValueIterator<Key,Value> iter : iterators) { if (iter.hasTop()) { return null; } } return data; }
public static Range maximizeStartKeyTimeStamp(Range range) { Range seekRange = range; if (range.getStartKey() != null) { Key seekKey = range.getStartKey(); if (range.getStartKey().getTimestamp() != Long.MAX_VALUE) { seekKey = new Key(seekRange.getStartKey()); seekKey.setTimestamp(Long.MAX_VALUE); seekRange = new Range(seekKey, true, range.getEndKey(), range.isEndKeyInclusive()); } else if (!range.isStartKeyInclusive()) { seekRange = new Range(seekKey, true, range.getEndKey(), range.isEndKeyInclusive()); } } return seekRange; }
TreeMap<Key,Value> consume(IteratorTestInput testInput, SortedKeyValueIterator<Key,Value> skvi, YieldCallback<Key> yield) throws IOException { TreeMap<Key,Value> data = new TreeMap<>(); Key lastKey = null; while (yield.hasYielded() || skvi.hasTop()) { if (yield.hasYielded()) { Range r = testInput.getRange(); Key yieldPosition = yield.getPositionAndReset(); if (!r.contains(yieldPosition)) { throw new IOException("Underlying iterator yielded to a position outside of its range: " + yieldPosition + " not in " + r); } if (skvi.hasTop()) { throw new IOException( "Underlying iterator reports having a top, but has yielded: " + yieldPosition); } if (lastKey != null && yieldPosition.compareTo(lastKey) <= 0) { throw new IOException( "Underlying iterator yielded at a position that is not past the last key returned"); } skvi.seek(new Range(yieldPosition, false, r.getEndKey(), r.isEndKeyInclusive()), testInput.getFamilies(), testInput.isInclusive()); } else { // Make sure to copy the K-V data.put(new Key(skvi.getTopKey()), new Value(skvi.getTopValue())); skvi.next(); } } return data; }
static boolean isRangeInBloomFilter(Range range, PartialKey keyDepth) { if (range.getStartKey() == null || range.getEndKey() == null) { return false; } if (range.getStartKey().equals(range.getEndKey(), keyDepth)) return true; // include everything but the deleted flag in the comparison... return range.getStartKey().followingKey(keyDepth).equals(range.getEndKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME) && !range.isEndKeyInclusive(); } }
/** * Converts the given {@code Range} into the correct {@code Range} for this TermSource (per this * expected table structure) and then seeks this TermSource's SKVI. */ public void seek(Range originalRange) throws IOException { // the infinite start key is equivalent to a null startKey on the Range. if (!originalRange.isInfiniteStartKey()) { Key originalStartKey = originalRange.getStartKey(); // Pivot the provided range into the range for this term Key newKey = new Key(originalStartKey.getRow(), term, originalStartKey.getColumnQualifier(), originalStartKey.getTimestamp()); // Construct the new range, preserving the other attributes on the provided range. currentRange = new Range(newKey, originalRange.isStartKeyInclusive(), originalRange.getEndKey(), originalRange.isEndKeyInclusive()); } else { currentRange = originalRange; } LOG.trace("Seeking {} to {}", this, currentRange); iter.seek(currentRange, seekColfams, true); }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { final String tableName = OptUtil.getTableOpt(cl, shellState); final ScanInterpreter interpeter = getInterpreter(cl, tableName, shellState); final Range range = getRange(cl, interpeter); final Authorizations auths = getAuths(cl, shellState); final Text startRow = range.getStartKey() == null ? null : range.getStartKey().getRow(); final Text endRow = range.getEndKey() == null ? null : range.getEndKey().getRow(); try { final Text max = shellState.getAccumuloClient().tableOperations().getMaxRow(tableName, auths, startRow, range.isStartKeyInclusive(), endRow, range.isEndKeyInclusive()); if (max != null) { shellState.getReader().println(max.toString()); } } catch (Exception e) { log.debug("Could not get shell state.", e); } return 0; }
public static Range minimizeEndKeyTimeStamp(Range range) { Range seekRange = range; if (range.getEndKey() != null) { Key seekKey = seekRange.getEndKey(); if (range.getEndKey().getTimestamp() != Long.MIN_VALUE) { seekKey = new Key(seekRange.getEndKey()); seekKey.setTimestamp(Long.MIN_VALUE); seekRange = new Range(range.getStartKey(), range.isStartKeyInclusive(), seekKey, true); } else if (!range.isEndKeyInclusive()) { seekRange = new Range(range.getStartKey(), range.isStartKeyInclusive(), seekKey, true); } } return seekRange; }
@Override protected void consume() throws IOException { if (finished || lastRowFound == null) return; int count = 0; SortedKeyValueIterator<Key,Value> source = getSource(); while (source.hasTop() && lastRowFound.equals(source.getTopKey().getRow())) { // try to efficiently jump to the next matching key if (count < numscans) { ++count; source.next(); // scan } else { // too many scans, just seek count = 0; // determine where to seek to, but don't go beyond the user-specified range Key nextKey = source.getTopKey().followingKey(PartialKey.ROW); if (!latestRange.afterEndKey(nextKey)) source.seek( new Range(nextKey, true, latestRange.getEndKey(), latestRange.isEndKeyInclusive()), latestColumnFamilies, latestInclusive); else { finished = true; break; } } } lastRowFound = source.hasTop() ? source.getTopKey().getRow(lastRowFound) : null; }
/** * Possibly expand {@code range} to include everything for the key prefix we are working with. * That is, if our prefix is ROW_COLFAM, then we need to expand the range so we're sure to include * all entries having the same row and column family as the start/end of the range. * * @param range * the range to expand * @return the modified range */ protected Range computeReseekRange(Range range) { Key startKey = range.getStartKey(); boolean startKeyInclusive = range.isStartKeyInclusive(); // If anything after the prefix is set, then clip the key so we include // everything for the prefix. if (isSetAfterPart(startKey, getKeyPrefix())) { startKey = copyPartialKey(startKey, getKeyPrefix()); startKeyInclusive = true; } Key endKey = range.getEndKey(); boolean endKeyInclusive = range.isEndKeyInclusive(); if (isSetAfterPart(endKey, getKeyPrefix())) { endKey = endKey.followingKey(getKeyPrefix()); endKeyInclusive = true; } return new Range(startKey, startKeyInclusive, endKey, endKeyInclusive); }
@Override public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { // save parameters for future internal seeks latestRange = range; latestColumnFamilies = columnFamilies; latestInclusive = inclusive; lastRowFound = null; Key startKey = range.getStartKey(); Range seekRange = new Range(startKey == null ? null : new Key(startKey.getRow()), true, range.getEndKey(), range.isEndKeyInclusive()); super.seek(seekRange, columnFamilies, inclusive); finished = false; if (getSource().hasTop()) { lastRowFound = getSource().getTopKey().getRow(); if (range.beforeStartKey(getSource().getTopKey())) consume(); } }
@Override public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { topKey = null; topValue = null; Key sk = range.getStartKey(); if (sk != null && sk.getColumnQualifierData().length() == 0 && sk.getColumnVisibilityData().length() == 0 && sk.getTimestamp() == Long.MAX_VALUE && !range.isStartKeyInclusive()) { // assuming that we are seeking using a key previously returned by // this iterator // therefore go to the next row/cf Key followingRowKey = sk.followingKey(PartialKey.ROW_COLFAM); if (range.getEndKey() != null && followingRowKey.compareTo(range.getEndKey()) > 0) return; range = new Range(sk.followingKey(PartialKey.ROW_COLFAM), true, range.getEndKey(), range.isEndKeyInclusive()); } sourceIter.seek(range, columnFamilies, inclusive); prepKeys(); }
@Override public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { topKey = null; topValue = null; Key sk = range.getStartKey(); if (sk != null && sk.getColumnFamilyData().length() == 0 && sk.getColumnQualifierData().length() == 0 && sk.getColumnVisibilityData().length() == 0 && sk.getTimestamp() == Long.MAX_VALUE && !range.isStartKeyInclusive()) { // assuming that we are seeking using a key previously returned by this iterator // therefore go to the next row Key followingRowKey = sk.followingKey(PartialKey.ROW); if (range.getEndKey() != null && followingRowKey.compareTo(range.getEndKey()) > 0) return; range = new Range(sk.followingKey(PartialKey.ROW), true, range.getEndKey(), range.isEndKeyInclusive()); } sourceIter.seek(range, columnFamilies, inclusive); prepKeys(); }