@Override public List<String> listSchemaNames(KuduClient client) { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(client); } rawSchemasTable = getSchemasTable(client); } KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
Vector<HashMap<String, ByteIterator>> result) { try { KuduScanner.KuduScannerBuilder scannerBuilder = client.newScannerBuilder(kuduTable); List<String> querySchema; if (fields == null) {
@Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { this.output = output; this.context = context; try { KuduTable table = client.openTable(scanSpec.getTableName()); KuduScannerBuilder builder = client.newScannerBuilder(table); if (!isStarQuery()) { List<String> colNames = Lists.newArrayList(); for (SchemaPath p : this.getColumns()) { colNames.add(p.getRootSegmentPath()); } builder.setProjectedColumnNames(colNames); } context.getStats().startWait(); try { scanner = builder .lowerBoundRaw(scanSpec.getStartKey()) .exclusiveUpperBoundRaw(scanSpec.getEndKey()) .build(); } finally { context.getStats().stopWait(); } } catch (Exception e) { throw new ExecutionSetupException(e); } }
KuduScanner scanner = client.newScannerBuilder(table) .setProjectedColumnNames(projectColumns) .build();
KuduScannerBuilder scannerBuilder = kuduClient.newScannerBuilder(table); JPQLExpression jpqlExp = kunderaQuery.getJpqlExpression(); List<String> selectColumns = KunderaQueryUtils.getSelectColumns(jpqlExp);
private RowResult nextNode(long keyOne, long keyTwo) throws KuduException { KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table); configureScannerForRandomRead(builder, table, keyOne, keyTwo); return getOneRowResult(builder.build()); } }
KuduScannerBuilder scannerBuilder = kuduClient.newScannerBuilder(table); KuduScanner scanner = null;
/** * Finds the next node in the linked list. */ private RowResult nextNode(long prevKeyOne, long prevKeyTwo) throws KuduException { KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table) .setProjectedColumnNames(SCAN_COLUMN_NAMES); configureScannerForRandomRead(builder, table, prevKeyOne, prevKeyTwo); return getOneRowResult(builder.build()); }
private KuduScanner scannerForFilters(Iterable<Row> filters, KuduTable table) throws KuduException { List<Row> filtersList = Lists.newArrayList(filters); if (filtersList.size() == 0) { throw new RuntimeException("Kudu existing filter was not provided."); } if (filtersList.get(0).schema() == null) { throw new RuntimeException("Kudu existing filter did not contain a schema."); } if (hasAccumulators()) { accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_SCANNERS).add(1); accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_FILTERS_SCANNED).add(filtersList.size()); } KuduScannerBuilder builder = getConnection().getClient().newScannerBuilder(table); for (String fieldName : filtersList.get(0).schema().fieldNames()) { ColumnSchema columnSchema = table.getSchema().getColumn(fieldName); List<Object> columnValues = Lists.newArrayList(); for (Row filter : filtersList) { Object columnValue = RowUtils.get(filter, fieldName); columnValues.add(columnValue); } KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, columnValues); builder = builder.addPredicate(predicate); } KuduScanner scanner = builder.build(); return scanner; }
@Override public List<String> listSchemaNames(KuduClient client) { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(client); } rawSchemasTable = getSchemasTable(client); } KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
private KuduScanner.KuduScannerBuilder getScannerBuilder() { return localClient.newScannerBuilder(table) .readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT) .snapshotTimestampRaw(sharedWriteTimestamp) .setFaultTolerant(true); }
@Override public List<String> listSchemaNames() { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(); } rawSchemasTable = getSchemasTable(); } ColumnSchema tenantColumn = rawSchemasTable.getSchema().getColumnByIndex(0); KuduScanner scanner = client.newScannerBuilder(rawSchemasTable) .addPredicate(KuduPredicate.newComparisonPredicate(tenantColumn, KuduPredicate.ComparisonOp.EQUAL, tenantPrefix)) .setProjectedColumnIndexes(ImmutableList.of(1)) .build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
public static List<String> scanTableToStrings(KuduTable table, KuduPredicate... predicates) throws Exception { List<String> rowStrings = Lists.newArrayList(); KuduScanner.KuduScannerBuilder scanBuilder = table.getAsyncClient().syncClient().newScannerBuilder(table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } KuduScanner scanner = scanBuilder.build(); while (scanner.hasMoreRows()) { RowResultIterator rows = scanner.nextRows(); for (RowResult r : rows) { rowStrings.add(r.rowToString()); } } Collections.sort(rowStrings); return rowStrings; }
/** * Scans the table and returns the number of rows. * @param table the table * @param predicates optional predicates to apply to the scan * @return the number of rows in the table matching the predicates */ public static long countRowsInTable(KuduTable table, KuduPredicate... predicates) throws KuduException { KuduScanner.KuduScannerBuilder scanBuilder = table.getAsyncClient().syncClient().newScannerBuilder(table); for (KuduPredicate predicate : predicates) { scanBuilder.addPredicate(predicate); } scanBuilder.setProjectedColumnIndexes(ImmutableList.<Integer>of()); return countRowsInScan(scanBuilder.build()); }
/** * Test for KUDU-1343 with a multi-batch multi-tablet scan. */ @Test(timeout = 100000) public void testKudu1343() throws Exception { KuduScanner scanner = harness.getClient().newScannerBuilder(table) .batchSizeBytes(1) // Just a hint, won't actually be that small .build(); int rowCount = 0; int loopCount = 0; while(scanner.hasMoreRows()) { loopCount++; RowResultIterator rri = scanner.nextRows(); rowCount += rri.getNumRows(); } assertTrue(loopCount > TABLET_COUNT); assertEquals(ROW_COUNT, rowCount); }
@Test(timeout = 100000) public void testBasicOps() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build())); OperationResponse resp = session.apply(createInsert(table, 0)); assertTrue(resp.hasRowError()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); for (int i = 10; i < 20; i++) { session.apply(createInsert(table, i)); } session.flush(); assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build())); }
@Test public void testConnectionRefused() throws Exception { CreateTableOptions options = getBasicCreateTableOptions(); KuduTable table = client.createTable( "testConnectionRefused-" + System.currentTimeMillis(), basicSchema, options); // Warm up the caches. assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build())); // Make it impossible to use Kudu. harness.killAllTabletServers(); // Create a scan with a short timeout. KuduScanner scanner = client.newScannerBuilder(table).scanRequestTimeout(1000).build(); // Check it fails. try { while (scanner.hasMoreRows()) { scanner.nextRows(); fail("The scan should timeout"); } } catch (NonRecoverableException ex) { assertTrue(ex.getStatus().isTimedOut()); } // Try the same thing with an insert. KuduSession session = client.newSession(); session.setTimeoutMillis(1000); OperationResponse response = session.apply(createBasicSchemaInsert(table, 1)); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isTimedOut()); }
/** * Counts the rows in a table between two optional bounds. * @param table the table to scan, must have the basic schema * @param lowerBound an optional lower bound key * @param upperBound an optional upper bound key * @return the row count * @throws Exception on error */ private int countRowsForTestScanNonCoveredTable(KuduTable table, Integer lowerBound, Integer upperBound) throws Exception { KuduScanner.KuduScannerBuilder scanBuilder = client.newScannerBuilder(table); if (lowerBound != null) { PartialRow bound = basicSchema.newPartialRow(); bound.addInt(0, lowerBound); scanBuilder.lowerBound(bound); } if (upperBound != null) { PartialRow bound = basicSchema.newPartialRow(); bound.addInt(0, upperBound); scanBuilder.exclusiveUpperBound(bound); } KuduScanner scanner = scanBuilder.build(); int count = 0; while (scanner.hasMoreRows()) { count += scanner.nextRows().getNumRows(); } return count; }
/** * Test setting and reading the most recent propagated timestamp. */ @Test(timeout = 100000) public void testLastPropagatedTimestamps() throws Exception { // Scan a table to ensure a timestamp is propagated. KuduTable table = client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); client.newScannerBuilder(table).build().nextRows().getNumRows(); assertTrue(client.hasLastPropagatedTimestamp()); assertTrue(client.hasLastPropagatedTimestamp()); assertTrue(asyncClient.hasLastPropagatedTimestamp()); long initial_ts = client.getLastPropagatedTimestamp(); // Check that the initial timestamp is consistent with the asynchronous client. assertEquals(initial_ts, client.getLastPropagatedTimestamp()); assertEquals(initial_ts, asyncClient.getLastPropagatedTimestamp()); // Attempt to change the timestamp to a lower value. This should not change // the internal timestamp, as it must be monotonically increasing. client.updateLastPropagatedTimestamp(initial_ts - 1); assertEquals(initial_ts, client.getLastPropagatedTimestamp()); assertEquals(initial_ts, asyncClient.getLastPropagatedTimestamp()); // Use the synchronous client to update the last propagated timestamp and // check with both clients that the timestamp was updated. client.updateLastPropagatedTimestamp(initial_ts + 1); assertEquals(initial_ts + 1, client.getLastPropagatedTimestamp()); assertEquals(initial_ts + 1, asyncClient.getLastPropagatedTimestamp()); }
@Test(timeout = 100000) public void testBatchWithSameRow() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end, // while also clearing the cache between each batch half the time. The delete is added here // so that a misplaced update would fail if it happens later than its delete. for (int i = 0; i < 25; i++) { session.apply(createInsert(table, i)); for (int j = 0; j < 50; j++) { Update update = table.newUpdate(); PartialRow row = update.getRow(); row.addInt(basicSchema.getColumnByIndex(0).getName(), i); row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000); session.apply(update); } Delete del = table.newDelete(); PartialRow row = del.getRow(); row.addInt(basicSchema.getColumnByIndex(0).getName(), i); session.apply(del); session.flush(); if (i % 2 == 0) { asyncClient.emptyTabletsCacheForTable(table.getTableId()); } } assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build())); }