log.debug("Fetched " + nextRows.getNumRows() + " rows");
@Override public RowResultIterator call(final Response resp) { numRowsReturned += resp.data.getNumRows(); if (!resp.more) { // We're done scanning this tablet. scanFinished(); return resp.data; } sequenceId++; hasMore = resp.more; //LOG.info("Scan.next is returning rows: " + resp.data.getNumRows()); return resp.data; }
private static RowResult getOneRowResult(KuduScanner scanner) throws KuduException { RowResultIterator rowResults; rowResults = scanner.nextRows(); if (rowResults.getNumRows() == 0) { return null; } if (rowResults.getNumRows() > 1) { throw new RuntimeException("Received too many rows from scanner " + scanner); } return rowResults.next(); }
@Override public Object call(RowResultIterator arg) throws Exception { if (arg == null) return null; counter.addAndGet(arg.getNumRows()); return null; } };
public static int countRowsInScan(KuduScanner scanner) throws KuduException { int counter = 0; while (scanner.hasMoreRows()) { counter += scanner.nextRows().getNumRows(); } return counter; }
numRowsReturned += resp.data.getNumRows();
/** * get next Row/Page */ @Override public boolean advanceNextPosition() { boolean needNextRows = !started || !nextRows.hasNext(); if (!started) { started = true; nanoStart = System.nanoTime(); } if (needNextRows) { currentRow = null; try { do { if (!scanner.hasMoreRows()) { return false; } nextRows = scanner.nextRows(); } while (!nextRows.hasNext()); log.debug("Fetched " + nextRows.getNumRows() + " rows"); } catch (KuduException e) { currentRow = null; throw new RuntimeException(e); } } currentRow = nextRows.next(); totalBytes += getRowLength(); return true; }
log.debug("Fetched " + nextRows.getNumRows() + " rows");
private int countRows(KuduTable table, KuduPredicate... predicates) throws Exception { KuduScanner.KuduScannerBuilder scanBuilder = new KuduScanner.KuduScannerBuilder(asyncClient, table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } KuduScanner scanner = scanBuilder.build(); int count = 0; while (scanner.hasMoreRows()) { count += scanner.nextRows().getNumRows(); } return count; }
/** * Test for KUDU-1343 with a multi-batch multi-tablet scan. */ @Test(timeout = 100000) public void testKudu1343() throws Exception { KuduScanner scanner = harness.getClient().newScannerBuilder(table) .batchSizeBytes(1) // Just a hint, won't actually be that small .build(); int rowCount = 0; int loopCount = 0; while(scanner.hasMoreRows()) { loopCount++; RowResultIterator rri = scanner.nextRows(); rowCount += rri.getNumRows(); } assertTrue(loopCount > TABLET_COUNT); assertEquals(ROW_COUNT, rowCount); }
assertEquals(0, scanner.nextRows().getNumRows());
int rows = scanner.nextRows().getNumRows(); assertTrue("Scanner did not read any rows", rows > 0);
loopCount++; RowResultIterator rri = scanner.nextRows(); rowCount += rri.getNumRows(); loopCount++; RowResultIterator rri = scanner.nextRows(); rowCount += rri.getNumRows();
/** * Counts the rows in a table between two optional bounds. * @param table the table to scan, must have the basic schema * @param lowerBound an optional lower bound key * @param upperBound an optional upper bound key * @return the row count * @throws Exception on error */ private int countRowsForTestScanNonCoveredTable(KuduTable table, Integer lowerBound, Integer upperBound) throws Exception { KuduScanner.KuduScannerBuilder scanBuilder = client.newScannerBuilder(table); if (lowerBound != null) { PartialRow bound = basicSchema.newPartialRow(); bound.addInt(0, lowerBound); scanBuilder.lowerBound(bound); } if (upperBound != null) { PartialRow bound = basicSchema.newPartialRow(); bound.addInt(0, upperBound); scanBuilder.exclusiveUpperBound(bound); } KuduScanner scanner = scanBuilder.build(); int count = 0; while (scanner.hasMoreRows()) { count += scanner.nextRows().getNumRows(); } return count; }
@Test(timeout = 100000) public void testReadAtSnapshotNoTimestamp() throws Exception { // Perform scan in READ_AT_SNAPSHOT mode with no snapshot timestamp // specified. Verify that the scanner timestamp is set from the tablet // server response. AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table) .readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT) .build(); assertEquals(AsyncKuduClient.NO_TIMESTAMP, scanner.getSnapshotTimestamp()); KuduScanner syncScanner = new KuduScanner(scanner); assertEquals(scanner.getReadMode(), syncScanner.getReadMode()); assertTrue(syncScanner.hasMoreRows()); assertEquals(AsyncKuduClient.NO_TIMESTAMP, scanner.getSnapshotTimestamp()); int rowCount = syncScanner.nextRows().getNumRows(); // At this point, the call to the first tablet server should have been // done already, so check the snapshot timestamp. final long tsRef = scanner.getSnapshotTimestamp(); assertNotEquals(AsyncKuduClient.NO_TIMESTAMP, tsRef); assertTrue(syncScanner.hasMoreRows()); while (syncScanner.hasMoreRows()) { rowCount += syncScanner.nextRows().getNumRows(); assertEquals(tsRef, scanner.getSnapshotTimestamp()); } assertEquals(9, rowCount); }
assertEquals(KuduClient.NO_TIMESTAMP, client.getLastPropagatedTimestamp()); int rowCount = syncScanner.nextRows().getNumRows(); rowCount += syncScanner.nextRows().getNumRows(); final long asyncTs = asyncClient.getLastPropagatedTimestamp(); final long syncTs = client.getLastPropagatedTimestamp();
@Test(timeout = 100000) public void testScanTokenPropagatesTimestamp() throws Exception { // Initially, the client does not have the timestamp set. assertEquals(AsyncKuduClient.NO_TIMESTAMP, asyncClient.getLastPropagatedTimestamp()); assertEquals(KuduClient.NO_TIMESTAMP, client.getLastPropagatedTimestamp()); AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table).build(); KuduScanner syncScanner = new KuduScanner(scanner); // Let the client receive the propagated timestamp in the scanner response. syncScanner.nextRows().getNumRows(); final long tsPrev = asyncClient.getLastPropagatedTimestamp(); final long tsPropagated = tsPrev + 1000000; ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setPropagatedTimestamp(tsPropagated); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize scan tokens and make sure the client's last propagated // timestamp is updated accordingly. assertEquals(tsPrev, asyncClient.getLastPropagatedTimestamp()); KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(tsPropagated, asyncClient.getLastPropagatedTimestamp()); }
int numRows = rri.join(DEFAULT_SLEEP).getNumRows(); assertNotEquals("The TS sent all the rows back, we can't properly test disconnection", rowCount, numRows);
/** * Test setting and reading the most recent propagated timestamp. */ @Test(timeout = 100000) public void testLastPropagatedTimestamps() throws Exception { // Scan a table to ensure a timestamp is propagated. KuduTable table = client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); client.newScannerBuilder(table).build().nextRows().getNumRows(); assertTrue(client.hasLastPropagatedTimestamp()); assertTrue(client.hasLastPropagatedTimestamp()); assertTrue(asyncClient.hasLastPropagatedTimestamp()); long initial_ts = client.getLastPropagatedTimestamp(); // Check that the initial timestamp is consistent with the asynchronous client. assertEquals(initial_ts, client.getLastPropagatedTimestamp()); assertEquals(initial_ts, asyncClient.getLastPropagatedTimestamp()); // Attempt to change the timestamp to a lower value. This should not change // the internal timestamp, as it must be monotonically increasing. client.updateLastPropagatedTimestamp(initial_ts - 1); assertEquals(initial_ts, client.getLastPropagatedTimestamp()); assertEquals(initial_ts, asyncClient.getLastPropagatedTimestamp()); // Use the synchronous client to update the last propagated timestamp and // check with both clients that the timestamp was updated. client.updateLastPropagatedTimestamp(initial_ts + 1); assertEquals(initial_ts + 1, client.getLastPropagatedTimestamp()); assertEquals(initial_ts + 1, asyncClient.getLastPropagatedTimestamp()); }
Deferred<RowResultIterator> d = scanner.nextRows(); RowResultIterator rri = d.join(DEFAULT_SLEEP); assertEquals(3, rri.getNumRows()); d = scanner.close(); rri = d.join(DEFAULT_SLEEP);