@Override public List<String> listSchemaNames(KuduClient client) { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(client); } rawSchemasTable = getSchemasTable(client); } KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
nextRows = scanner.nextRows();
RowResultIterator data = scanner.nextRows(); addAllRowsToResult(data, recordcount, querySchema, result); if (recordcount == result.size()) {
@Override public int next() { int rowCount = 0; try { while (iterator == null || !iterator.hasNext()) { if (!scanner.hasMoreRows()) { iterator = null; return 0; } context.getStats().startWait(); try { iterator = scanner.nextRows(); } finally { context.getStats().stopWait(); } } for (; rowCount < TARGET_RECORD_COUNT && iterator.hasNext(); rowCount++) { addRowResult(iterator.next(), rowCount); } } catch (Exception ex) { throw new RuntimeException(ex); } for (ProjectedColumnInfo pci : projectedCols) { pci.vv.getMutator().setValueCount(rowCount); } return rowCount; }
.build(); while (scanner.hasMoreRows()) { RowResultIterator results = scanner.nextRows(); while (results.hasNext()) { logger.debug(results.next().toString());
try rowResultIter = scanner.nextRows();
try results = scanner.nextRows();
private static RowResult getOneRowResult(KuduScanner scanner) throws KuduException { RowResultIterator rowResults; rowResults = scanner.nextRows(); if (rowResults.getNumRows() == 0) { return null; } if (rowResults.getNumRows() > 1) { throw new RuntimeException("Received too many rows from scanner " + scanner); } return rowResults.next(); }
@Override public void open(KuduInputSplit split) throws IOException { endReached = false; startTableContext(); scanner = tableContext.scanner(split.getScanToken()); resultIterator = scanner.nextRows(); }
private Set<Row> collectRows(KuduScanner scanner) throws KuduException { Set<Row> rows = new HashSet<>(); while (scanner.hasMoreRows()) { for (RowResult result : scanner.nextRows()) { rows.add(Row.fromResult(result)); } } return rows; }
@Override public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception { List<Row> existingForFilters = Lists.newArrayList(); if (!filters.iterator().hasNext()) { return existingForFilters; } KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME)); KuduScanner scanner = scannerForFilters(filters, table); long startTime = System.nanoTime(); while (scanner.hasMoreRows()) { for (RowResult rowResult : scanner.nextRows()) { Row existing = resultAsRow(rowResult, table); existingForFilters.add(existing); } } long endTime = System.nanoTime(); if (hasAccumulators()) { accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0); } return existingForFilters; }
@Override public List<String> listSchemaNames(KuduClient client) { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(client); } rawSchemasTable = getSchemasTable(client); } KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
public static int countRowsInScan(KuduScanner scanner) throws KuduException { int counter = 0; while (scanner.hasMoreRows()) { counter += scanner.nextRows().getNumRows(); } return counter; }
@Override public KuduRow nextRecord(KuduRow reuse) throws IOException { // check that current iterator has next rows if (this.resultIterator.hasNext()) { RowResult row = this.resultIterator.next(); return KuduMapper.toKuduRow(row); } // if not, check that current scanner has more iterators else if (scanner.hasMoreRows()) { this.resultIterator = scanner.nextRows(); return nextRecord(reuse); } else { endReached = true; } return null; }
public static List<String> scanTableToStrings(KuduTable table, KuduPredicate... predicates) throws Exception { List<String> rowStrings = Lists.newArrayList(); KuduScanner.KuduScannerBuilder scanBuilder = table.getAsyncClient().syncClient().newScannerBuilder(table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } KuduScanner scanner = scanBuilder.build(); while (scanner.hasMoreRows()) { RowResultIterator rows = scanner.nextRows(); for (RowResult r : rows) { rowStrings.add(r.rowToString()); } } Collections.sort(rowStrings); return rowStrings; }
private int countRows(KuduTable table, KuduPredicate... predicates) throws Exception { KuduScanner.KuduScannerBuilder scanBuilder = new KuduScanner.KuduScannerBuilder(asyncClient, table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } KuduScanner scanner = scanBuilder.build(); int count = 0; while (scanner.hasMoreRows()) { count += scanner.nextRows().getNumRows(); } return count; }
@Override public void run() { try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses) .defaultAdminOperationTimeoutMs(operationTimeoutMs) .build()) { KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient); try { int localCount = 0; while (scanner.hasMoreRows()) { localCount += Iterators.size(scanner.nextRows()); } count.addAndGet(localCount); } finally { scanner.close(); } } catch (Exception e) { LOG.error("exception in parallel token scanner", e); } } });
/** * Test for KUDU-1343 with a multi-batch multi-tablet scan. */ @Test(timeout = 100000) public void testKudu1343() throws Exception { KuduScanner scanner = harness.getClient().newScannerBuilder(table) .batchSizeBytes(1) // Just a hint, won't actually be that small .build(); int rowCount = 0; int loopCount = 0; while(scanner.hasMoreRows()) { loopCount++; RowResultIterator rri = scanner.nextRows(); rowCount += rri.getNumRows(); } assertTrue(loopCount > TABLET_COUNT); assertEquals(ROW_COUNT, rowCount); }
@Test(timeout = 100000) public void testReadAtSnapshotNoTimestamp() throws Exception { // Perform scan in READ_AT_SNAPSHOT mode with no snapshot timestamp // specified. Verify that the scanner timestamp is set from the tablet // server response. AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table) .readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT) .build(); assertEquals(AsyncKuduClient.NO_TIMESTAMP, scanner.getSnapshotTimestamp()); KuduScanner syncScanner = new KuduScanner(scanner); assertEquals(scanner.getReadMode(), syncScanner.getReadMode()); assertTrue(syncScanner.hasMoreRows()); assertEquals(AsyncKuduClient.NO_TIMESTAMP, scanner.getSnapshotTimestamp()); int rowCount = syncScanner.nextRows().getNumRows(); // At this point, the call to the first tablet server should have been // done already, so check the snapshot timestamp. final long tsRef = scanner.getSnapshotTimestamp(); assertNotEquals(AsyncKuduClient.NO_TIMESTAMP, tsRef); assertTrue(syncScanner.hasMoreRows()); while (syncScanner.hasMoreRows()) { rowCount += syncScanner.nextRows().getNumRows(); assertEquals(tsRef, scanner.getSnapshotTimestamp()); } assertEquals(9, rowCount); }
@Test(timeout = 100000) public void testScanTokenPropagatesTimestamp() throws Exception { // Initially, the client does not have the timestamp set. assertEquals(AsyncKuduClient.NO_TIMESTAMP, asyncClient.getLastPropagatedTimestamp()); assertEquals(KuduClient.NO_TIMESTAMP, client.getLastPropagatedTimestamp()); AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table).build(); KuduScanner syncScanner = new KuduScanner(scanner); // Let the client receive the propagated timestamp in the scanner response. syncScanner.nextRows().getNumRows(); final long tsPrev = asyncClient.getLastPropagatedTimestamp(); final long tsPropagated = tsPrev + 1000000; ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setPropagatedTimestamp(tsPropagated); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize scan tokens and make sure the client's last propagated // timestamp is updated accordingly. assertEquals(tsPrev, asyncClient.getLastPropagatedTimestamp()); KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(tsPropagated, asyncClient.getLastPropagatedTimestamp()); }