@Override public void init() throws DBException { this.tableName = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); initClient(); this.session = client.newSession(); if (getProperties().getProperty(SYNC_OPS_OPT) != null && getProperties().getProperty(SYNC_OPS_OPT).equals("false")) { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); this.session.setMutationBufferSpace( getIntFromProp(getProperties(), BUFFER_NUM_OPS_OPT, BUFFER_NUM_OPS_DEFAULT)); } else { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_SYNC); } try { this.kuduTable = client.openTable(tableName); this.schema = kuduTable.getSchema(); } catch (Exception e) { throw new DBException("Could not open a table because of:", e); } }
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { try { String rawName = schemaEmulation.toRawName(tableMetadata.getTable()); if (ignoreExisting) { if (client.tableExists(rawName)) { return null; } } if (!schemaEmulation.existsSchema(client, tableMetadata.getTable().getSchemaName())) { throw new SchemaNotFoundException(tableMetadata.getTable().getSchemaName()); } List<ColumnMetadata> columns = tableMetadata.getColumns(); Map<String, Object> properties = tableMetadata.getProperties(); Schema schema = buildSchema(columns, properties); CreateTableOptions options = buildCreateTableOptions(schema, properties); return client.createTable(rawName, schema, options); } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
@Override public List<String> listSchemaNames(KuduClient client) { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(client); } rawSchemasTable = getSchemasTable(client); } KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
@Override public void dropSchema(KuduClient client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { throw new PrestoException(GENERIC_USER_ERROR, "Deleting default schema not allowed."); } else { try { String prefix = getPrefixForTablesOfSchema(schemaName); for (String name : client.getTablesList(prefix).getTablesList()) { client.deleteTable(name); } KuduTable schemasTable = getSchemasTable(client); KuduSession session = client.newSession(); try { Delete delete = schemasTable.newDelete(); delete.getRow().addString(0, schemaName); session.apply(delete); } finally { session.close(); } } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } } }
final String TABLE_NAME = TestClientFailoverSupport.class.getName() + "-" + failureType; client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); KuduTable table = client.openTable(TABLE_NAME); KuduSession session = client.newSession(); client.deleteTable(TABLE_NAME); assertFalse(client.tableExists(TABLE_NAME));
ListTablesResponse tables = client.getTablesList(tableName); if (!tables.getTablesList().isEmpty()) { client.deleteTable(tableName); client.createTable(tableName, schema, builder); KuduTable table = client.openTable(tableName); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); for (int i = 0; i < rows; i++) { KuduScanner scanner = client.newScannerBuilder(table) .setProjectedColumnNames(projectColumns) .build();
@Override @SuppressWarnings("AssertionFailureIgnored") public void run() { final String tableName = "TestAuthnTokenReacquire-table-" + threadIdx; try { ListTabletServersResponse response = client.listTabletServers(); assertNotNull(response); dropConnectionsAndExpireToken(); ListTablesResponse tableList = client.getTablesList(tableName); assertNotNull(tableList); assertTrue(tableList.getTablesList().isEmpty()); dropConnectionsAndExpireToken(); client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); dropConnectionsAndExpireToken(); KuduTable table = client.openTable(tableName); assertEquals(basicSchema.getColumnCount(), table.getSchema().getColumnCount()); dropConnectionsAndExpireToken(); client.deleteTable(tableName); assertFalse(client.tableExists(tableName)); } catch (Throwable e) { //noinspection ThrowableResultOfMethodCallIgnored exceptions.put(threadIdx, e); } } });
options.addRangePartition(lower, upper); client.createTable(TABLE_NAME, basicSchema, options); KuduTable table = client.openTable(TABLE_NAME); KuduScanToken.KuduScanTokenBuilder tokenBuilder = client.newScanTokenBuilder(table); List<KuduScanToken> tokens = tokenBuilder.build(); assertEquals(1, tokens.size()); lower.addInt("key", 1); alter.addRangePartition(lower, upper); alterClient.alterTable(TABLE_NAME, alter); tokenBuilder = client.newScanTokenBuilder(table); tokens = tokenBuilder.build(); assertEquals(1, tokens.size()); table = client.openTable(TABLE_NAME); tokenBuilder = client.newScanTokenBuilder(table); tokens = tokenBuilder.build(); assertEquals(2, tokens.size());
createOptions.addSplitRow(split); client.createTable(testTableName, schema, createOptions); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); KuduTable table = client.openTable(testTableName); for (char c = 'a'; c < 'f'; c++) { Insert insert = table.newInsert(); KuduScanToken.KuduScanTokenBuilder tokenBuilder = client.newScanTokenBuilder(table); tokenBuilder.setProjectedColumnIndexes(ImmutableList.<Integer>of()); List<KuduScanToken> tokens = tokenBuilder.build(); assertEquals('f' - 'a' + 'z' - 'h', countScanTokenRows(tokens, client.getMasterAddressesAsString(), client.getDefaultOperationTimeoutMs()));
@Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { this.output = output; this.context = context; try { KuduTable table = client.openTable(scanSpec.getTableName()); KuduScannerBuilder builder = client.newScannerBuilder(table); if (!isStarQuery()) { List<String> colNames = Lists.newArrayList(); for (SchemaPath p : this.getColumns()) { colNames.add(p.getRootSegmentPath()); } builder.setProjectedColumnNames(colNames); } context.getStats().startWait(); try { scanner = builder .lowerBoundRaw(scanSpec.getStartKey()) .exclusiveUpperBoundRaw(scanSpec.getEndKey()) .build(); } finally { context.getStats().stopWait(); } } catch (Exception e) { throw new ExecutionSetupException(e); } }
@Test(timeout = 100000) public void testFormatRangePartitionsUnbounded() throws Exception { CreateTableOptions builder = getBasicCreateTableOptions(); client.createTable(tableName, basicSchema, builder); assertEquals( ImmutableList.of("UNBOUNDED"), client.openTable(tableName).getFormattedRangePartitions(10000)); }
createOptions.addSplitRow(splitRow); client.createTable(testTableName, schema, createOptions); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); KuduTable table = client.openTable(testTableName); for (int i = 0; i < 100; i++) { Insert insert = table.newInsert(); KuduScanToken.KuduScanTokenBuilder tokenBuilder = client.newScanTokenBuilder(table); tokenBuilder.batchSizeBytes(0); tokenBuilder.setProjectedColumnIndexes(ImmutableList.<Integer>of());
client.createTable(tableName, schema, tableBuilder); KuduTable table = client.openTable(tableName); assertEquals(rows, collectRows(client.newScannerBuilder(table).build())); KuduScanner scanner = client.newScannerBuilder(table).lowerBound(lowerBound).build(); Set<Row> results = collectRows(scanner); assertEquals(expected, results); client.newScanTokenBuilder(table).lowerBound(lowerBound); Set<Row> tokenResults = collectRows(scanTokens); assertEquals(expected, tokenResults); KuduScanner scanner = client.newScannerBuilder(table) .exclusiveUpperBound(upperBound) .build(); client.newScanTokenBuilder(table).exclusiveUpperBound(upperBound); Set<Row> tokenResults = collectRows(scanTokens); assertEquals(expected, tokenResults); KuduScanner scanner = client.newScannerBuilder(table) .lowerBound(lowerBound) .exclusiveUpperBound(upperBound) client.newScanTokenBuilder(table) .lowerBound(lowerBound) .exclusiveUpperBound(upperBound);
/** * Test creating and deleting a table through a KuduClient. */ @Test(timeout = 100000) public void testCreateDeleteTable() throws Exception { // Check that we can create a table. client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); assertFalse(client.getTablesList().getTablesList().isEmpty()); assertTrue(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can delete it. client.deleteTable(TABLE_NAME); assertFalse(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can re-recreate it, with a different schema. List<ColumnSchema> columns = new ArrayList<>(basicSchema.getColumns()); columns.add(new ColumnSchema.ColumnSchemaBuilder("one more", Type.STRING).build()); Schema newSchema = new Schema(columns); client.createTable(TABLE_NAME, newSchema, getBasicCreateTableOptions()); // Check that we can open a table and see that it has the new schema. KuduTable table = client.openTable(TABLE_NAME); assertEquals(newSchema.getColumnCount(), table.getSchema().getColumnCount()); assertTrue(table.getPartitionSchema().isSimpleRangePartitioning()); // Check that the block size parameter we specified in the schema is respected. assertEquals(4096, newSchema.getColumn("column3_s").getDesiredBlockSize()); assertEquals(ColumnSchema.Encoding.DICT_ENCODING, newSchema.getColumn("column3_s").getEncoding()); assertEquals(ColumnSchema.CompressionAlgorithm.LZ4, newSchema.getColumn("column3_s").getCompressionAlgorithm()); }
@Test(timeout = 100000) public void testBasicOps() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build())); OperationResponse resp = session.apply(createInsert(table, 0)); assertTrue(resp.hasRowError()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); for (int i = 10; i < 20; i++) { session.apply(createInsert(table, i)); } session.flush(); assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build())); }
public KuduSession newSession() { return client.newSession(); }
private KuduTable table(KuduTableInfo infoTable) throws IOException { KuduClient syncClient = client.syncClient(); String tableName = infoTable.getName(); if (syncClient.tableExists(tableName)) { return syncClient.openTable(tableName); } if (infoTable.createIfNotExist()) { return syncClient.createTable(tableName, infoTable.getSchema(), infoTable.getCreateTableOptions()); } throw new UnsupportedOperationException("table not exists and is marketed to not be created"); }
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
private KuduTable getSchemasTable(KuduClient client) throws KuduException { if (rawSchemasTable == null) { rawSchemasTable = client.openTable(rawSchemasTableName); } return rawSchemasTable; }
client.createTable(tableName, schema, builder); } catch (Exception e) { if (!e.getMessage().contains("already exists")) {