boolean result = super.next(outResult, ctx); if (!rawMobScan) {
@Test public void testSkipColumn() throws IOException { List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); assertEquals(kvs[0], results.get(0)); assertEquals(kvs[3], results.get(1)); results.clear(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[kvs.length - 1], results.get(0)); results.clear(); assertEquals(false, scan.next(results)); } }
/** * Gets the number of del cell in the del files * @param paths the del file paths * @return the cell size */ private int countDelCellsInDelFiles(List<Path> paths) throws IOException { List<HStoreFile> sfs = new ArrayList<>(); int size = 0; for (Path path : paths) { HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); sfs.add(sf); } List<KeyValueScanner> scanners = new ArrayList<>(StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, false, HConstants.LATEST_TIMESTAMP)); long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0); long ttl = HStore.determineTTLFromFamily(hcd); ScanInfo scanInfo = new ScanInfo(conf, hcd, ttl, timeToPurgeDeletes, CellComparatorImpl.COMPARATOR); StoreScanner scanner = new StoreScanner(scanInfo, ScanType.COMPACT_RETAIN_DELETES, scanners); List<Cell> results = new ArrayList<>(); boolean hasMore = true; while (hasMore) { hasMore = scanner.next(results); size += results.size(); results.clear(); } scanner.close(); return size; }
@Test public void testDeletedRowThenGoodRow() throws IOException { KeyValue [] kvs = new KeyValue [] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(0, results.size()); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[2], results.get(0)); assertEquals(false, scan.next(results)); } }
@Test public void testWontNextToNext() throws IOException { // build the scan file: KeyValue [] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); // this only uses maxVersions (default=1) and TimeRange (default=all) try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); scan.next(results); assertEquals(1, results.size()); assertEquals(kvs[0], results.get(0)); // should be ok... // now scan _next_ again. results.clear(); scan.next(results); assertEquals(1, results.size()); assertEquals(kvs[2], results.get(0)); results.clear(); scan.next(results); assertEquals(0, results.size()); } }
ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); do { hasMore = scanner.next(cells, scannerContext); for (Cell cell : cells) { writer.append(cell);
@Test public void testWildCardOneVersionScan() throws IOException { KeyValue [] kvs = new KeyValue [] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); assertEquals(kvs[0], results.get(0)); assertEquals(kvs[1], results.get(1)); } }
@Test public void testDeleteVersionSameTimestamp() throws IOException { KeyValue [] kvs = new KeyValue [] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertFalse(scan.next(results)); assertEquals(0, results.size()); } }
@Test public void testDeleteColumn() throws IOException { KeyValue [] kvs = new KeyValue[] { create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[3], results.get(0)); } }
/** * Ensure that expired delete family markers don't override valid puts */ @Test public void testExpiredDeleteFamily() throws Exception { long now = System.currentTimeMillis(); KeyValue[] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, KeyValue.Type.DeleteFamily), create("R1", "cf", "a", now-10, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scan = new Scan(); scan.readVersions(1); // scanner with ttl equal to 500 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scanner.next(results)); assertEquals(1, results.size()); assertEquals(kvs[1], results.get(0)); results.clear(); assertEquals(false, scanner.next(results)); } }
@Test public void testScanSameTimestamp() throws IOException { // returns only 1 of these 2 even though same timestamp KeyValue [] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = Arrays.asList( new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) }); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); // this only uses maxVersions (default=1) and TimeRange (default=all) try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[0], results.get(0)); } }
do { List<Cell> cells = new ArrayList<>(); more = s.next(cells); cellCount += cells.size(); assertEquals(more ? numberOfMemScannersAfterFlush : 0, countMemStoreScanner(s));
try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scanner.next(results)); assertEquals(2, results.size()); assertEquals(kvs[1], results.get(0)); results.clear(); assertEquals(true, scanner.next(results)); assertEquals(3, results.size()); assertEquals(kvs[4], results.get(0)); results.clear(); assertEquals(false, scanner.next(results));
@Test public void testDeleteVersionMaskingMultiplePuts() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); // the two put at ts=now will be masked by the 1 delete, and // since the scan default returns 1 version we'll return the newest // key, which is kvs[2], now-100. assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs2[1], results.get(0)); } }
@Test public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); assertEquals(kvs2[1], results.get(0)); assertEquals(kvs2[0], results.get(1)); } }
int count = 0; try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { while (s.next(result)) { LOG.info(Objects.toString(result)); count++; while (s.next(result)) { LOG.info(Objects.toString(result)); int snapshotIndex = 5; try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { while (s.next(result)) { LOG.info(Objects.toString(result));
@Test public void testDeleteFamily() throws IOException { KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(0, results.size()); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[kvs.length - 1], results.get(0)); assertEquals(false, scan.next(results)); } }
try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(5, results.size()); assertEquals(kvs[kvs.length - 1], results.get(0)); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(3, results.size());
new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(5, results.size()); assertEquals(kvs[0], results.get(0));
List<Cell> results = new ArrayList<>(); results = new ArrayList<>(); assertEquals(true, scanner.next(results)); assertEquals(kvs[0], results.get(0)); assertEquals(kvs[2], results.get(1));