try { do { if (!scanner.hasMoreRows()) { return false;
while (scanner.hasMoreRows()) { RowResultIterator data = scanner.nextRows(); addAllRowsToResult(data, recordcount, querySchema, result);
@Override public int next() { int rowCount = 0; try { while (iterator == null || !iterator.hasNext()) { if (!scanner.hasMoreRows()) { iterator = null; return 0; } context.getStats().startWait(); try { iterator = scanner.nextRows(); } finally { context.getStats().stopWait(); } } for (; rowCount < TARGET_RECORD_COUNT && iterator.hasNext(); rowCount++) { addRowResult(iterator.next(), rowCount); } } catch (Exception ex) { throw new RuntimeException(ex); } for (ProjectedColumnInfo pci : projectedCols) { pci.vv.getMutator().setValueCount(rowCount); } return rowCount; }
.setProjectedColumnNames(projectColumns) .build(); while (scanner.hasMoreRows()) { RowResultIterator results = scanner.nextRows(); while (results.hasNext()) {
while (scanner.hasMoreRows())
while (scanner.hasMoreRows())
private Set<Row> collectRows(KuduScanner scanner) throws KuduException { Set<Row> rows = new HashSet<>(); while (scanner.hasMoreRows()) { for (RowResult result : scanner.nextRows()) { rows.add(Row.fromResult(result)); } } return rows; }
@Override public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception { List<Row> existingForFilters = Lists.newArrayList(); if (!filters.iterator().hasNext()) { return existingForFilters; } KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME)); KuduScanner scanner = scannerForFilters(filters, table); long startTime = System.nanoTime(); while (scanner.hasMoreRows()) { for (RowResult rowResult : scanner.nextRows()) { Row existing = resultAsRow(rowResult, table); existingForFilters.add(existing); } } long endTime = System.nanoTime(); if (hasAccumulators()) { accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0); } return existingForFilters; }
public static int countRowsInScan(KuduScanner scanner) throws KuduException { int counter = 0; while (scanner.hasMoreRows()) { counter += scanner.nextRows().getNumRows(); } return counter; }
@Override public KuduRow nextRecord(KuduRow reuse) throws IOException { // check that current iterator has next rows if (this.resultIterator.hasNext()) { RowResult row = this.resultIterator.next(); return KuduMapper.toKuduRow(row); } // if not, check that current scanner has more iterators else if (scanner.hasMoreRows()) { this.resultIterator = scanner.nextRows(); return nextRecord(reuse); } else { endReached = true; } return null; }
/** * get next Row/Page */ @Override public boolean advanceNextPosition() { boolean needNextRows = !started || !nextRows.hasNext(); if (!started) { started = true; nanoStart = System.nanoTime(); } if (needNextRows) { currentRow = null; try { do { if (!scanner.hasMoreRows()) { return false; } nextRows = scanner.nextRows(); } while (!nextRows.hasNext()); log.debug("Fetched " + nextRows.getNumRows() + " rows"); } catch (KuduException e) { currentRow = null; throw new RuntimeException(e); } } currentRow = nextRows.next(); totalBytes += getRowLength(); return true; }
public static List<String> scanTableToStrings(KuduTable table, KuduPredicate... predicates) throws Exception { List<String> rowStrings = Lists.newArrayList(); KuduScanner.KuduScannerBuilder scanBuilder = table.getAsyncClient().syncClient().newScannerBuilder(table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } KuduScanner scanner = scanBuilder.build(); while (scanner.hasMoreRows()) { RowResultIterator rows = scanner.nextRows(); for (RowResult r : rows) { rowStrings.add(r.rowToString()); } } Collections.sort(rowStrings); return rowStrings; }
while (scanner.hasMoreRows()) { try { RowResultIterator ite = scanner.nextRows();
@Override public void run() { try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses) .defaultAdminOperationTimeoutMs(operationTimeoutMs) .build()) { KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient); try { int localCount = 0; while (scanner.hasMoreRows()) { localCount += Iterators.size(scanner.nextRows()); } count.addAndGet(localCount); } finally { scanner.close(); } } catch (Exception e) { LOG.error("exception in parallel token scanner", e); } } });
private int countRows(KuduTable table, KuduPredicate... predicates) throws Exception { KuduScanner.KuduScannerBuilder scanBuilder = new KuduScanner.KuduScannerBuilder(asyncClient, table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } KuduScanner scanner = scanBuilder.build(); int count = 0; while (scanner.hasMoreRows()) { count += scanner.nextRows().getNumRows(); } return count; }
@Test(timeout = 100000) public void testReadAtSnapshotNoTimestamp() throws Exception { // Perform scan in READ_AT_SNAPSHOT mode with no snapshot timestamp // specified. Verify that the scanner timestamp is set from the tablet // server response. AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table) .readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT) .build(); assertEquals(AsyncKuduClient.NO_TIMESTAMP, scanner.getSnapshotTimestamp()); KuduScanner syncScanner = new KuduScanner(scanner); assertEquals(scanner.getReadMode(), syncScanner.getReadMode()); assertTrue(syncScanner.hasMoreRows()); assertEquals(AsyncKuduClient.NO_TIMESTAMP, scanner.getSnapshotTimestamp()); int rowCount = syncScanner.nextRows().getNumRows(); // At this point, the call to the first tablet server should have been // done already, so check the snapshot timestamp. final long tsRef = scanner.getSnapshotTimestamp(); assertNotEquals(AsyncKuduClient.NO_TIMESTAMP, tsRef); assertTrue(syncScanner.hasMoreRows()); while (syncScanner.hasMoreRows()) { rowCount += syncScanner.nextRows().getNumRows(); assertEquals(tsRef, scanner.getSnapshotTimestamp()); } assertEquals(9, rowCount); }
/** * Test for KUDU-1343 with a multi-batch multi-tablet scan. */ @Test(timeout = 100000) public void testKudu1343() throws Exception { KuduScanner scanner = harness.getClient().newScannerBuilder(table) .batchSizeBytes(1) // Just a hint, won't actually be that small .build(); int rowCount = 0; int loopCount = 0; while(scanner.hasMoreRows()) { loopCount++; RowResultIterator rri = scanner.nextRows(); rowCount += rri.getNumRows(); } assertTrue(loopCount > TABLET_COUNT); assertEquals(ROW_COUNT, rowCount); }
assertTrue(syncScanner.hasMoreRows()); assertEquals(AsyncKuduClient.NO_TIMESTAMP, asyncClient.getLastPropagatedTimestamp()); assertEquals(KuduClient.NO_TIMESTAMP, client.getLastPropagatedTimestamp()); assertNotEquals(KuduClient.NO_TIMESTAMP, syncTsRef); assertTrue(syncScanner.hasMoreRows()); while (syncScanner.hasMoreRows()) { rowCount += syncScanner.nextRows().getNumRows(); final long asyncTs = asyncClient.getLastPropagatedTimestamp();
/** * Counts the rows in a table between two optional bounds. * @param table the table to scan, must have the basic schema * @param lowerBound an optional lower bound key * @param upperBound an optional upper bound key * @return the row count * @throws Exception on error */ private int countRowsForTestScanNonCoveredTable(KuduTable table, Integer lowerBound, Integer upperBound) throws Exception { KuduScanner.KuduScannerBuilder scanBuilder = client.newScannerBuilder(table); if (lowerBound != null) { PartialRow bound = basicSchema.newPartialRow(); bound.addInt(0, lowerBound); scanBuilder.lowerBound(bound); } if (upperBound != null) { PartialRow bound = basicSchema.newPartialRow(); bound.addInt(0, upperBound); scanBuilder.exclusiveUpperBound(bound); } KuduScanner scanner = scanBuilder.build(); int count = 0; while (scanner.hasMoreRows()) { count += scanner.nextRows().getNumRows(); } return count; }
@Test public void testConnectionRefused() throws Exception { CreateTableOptions options = getBasicCreateTableOptions(); KuduTable table = client.createTable( "testConnectionRefused-" + System.currentTimeMillis(), basicSchema, options); // Warm up the caches. assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build())); // Make it impossible to use Kudu. harness.killAllTabletServers(); // Create a scan with a short timeout. KuduScanner scanner = client.newScannerBuilder(table).scanRequestTimeout(1000).build(); // Check it fails. try { while (scanner.hasMoreRows()) { scanner.nextRows(); fail("The scan should timeout"); } } catch (NonRecoverableException ex) { assertTrue(ex.getStatus().isTimedOut()); } // Try the same thing with an insert. KuduSession session = client.newSession(); session.setTimeoutMillis(1000); OperationResponse response = session.apply(createBasicSchemaInsert(table, 1)); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isTimedOut()); }