private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties) { CreateTableOptions options = new CreateTableOptions(); RangePartitionDefinition rangePartitionDefinition = null; PartitionDesign partitionDesign = KuduTableProperties.getPartitionDesign(properties); if (partitionDesign.getHash() != null) { for (HashPartitionDefinition partition : partitionDesign.getHash()) { options.addHashPartitions(partition.getColumns(), partition.getBuckets()); } } if (partitionDesign.getRange() != null) { rangePartitionDefinition = partitionDesign.getRange(); options.setRangePartitionColumns(rangePartitionDefinition.getColumns()); } List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties); if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) { for (RangePartition rangePartition : rangePartitions) { PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower()); PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper()); options.addRangePartition(lower, upper); } } Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties); numReplicas.ifPresent(options::setNumReplicas); return options; }
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
try CreateTableOptions builder = new CreateTableOptions(); builder.setRangePartitionColumns(rangeKeys); client.createTable(tableInfo.getTableName(), schema, builder); logger.debug("Table: " + tableInfo.getTableName() + " created successfully");
@Test(timeout = 100000) public void testSimplePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("a", "b", "c")); PartialRow split = schema.newPartialRow(); split.addString("c", "3"); tableBuilder.addSplitRow(split); split = schema.newPartialRow(); split.addString("c", "3"); split.addString("b", "3"); tableBuilder.addSplitRow(split); testPartitionSchema(tableBuilder); }
@Test(timeout = 100000) public void testHashBucketedTable() throws Exception { CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.addHashPartitions(ImmutableList.of("a"), 3); tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42); tableBuilder.setRangePartitionColumns(ImmutableList.<String>of()); testPartitionSchema(tableBuilder); }
@Before public void setUp() throws Exception { String runtimeProp = System.getProperty(RUNTIME_PROPERTY_NAME); runtimeInSeconds = runtimeProp == null ? DEFAULT_RUNTIME_SECONDS : Long.parseLong(runtimeProp); if (runtimeInSeconds < TEST_MIN_RUNTIME_SECONDS || runtimeInSeconds > TEST_TIMEOUT_SECONDS) { Assert.fail("This test needs to run more than " + TEST_MIN_RUNTIME_SECONDS + " seconds" + " and less than " + TEST_TIMEOUT_SECONDS + " seconds"); } LOG.info ("Test running for {} seconds", runtimeInSeconds); // Client we're using has low tolerance for read timeouts but a // higher overall operation timeout. localAsyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(harness.getMasterAddressesAsString()) .defaultSocketReadTimeoutMs(500) .build(); localClient = new KuduClient(localAsyncClient); CreateTableOptions builder = new CreateTableOptions().setNumReplicas(3); builder.setRangePartitionColumns(ImmutableList.of("key")); table = localClient.createTable(TABLE_NAME, getBasicSchema(), builder); }
Schema schema = new Schema(columns); CreateTableOptions builder = new CreateTableOptions(); builder.setRangePartitionColumns(ImmutableList.of("a")); List<String> expected = Lists.newArrayList(); PartialRow upper = schema.newPartialRow(); upper.addString(0, "\0"); builder.addRangePartition(schema.newPartialRow(), upper); PartialRow upper = schema.newPartialRow(); upper.addString(0, "abc\0"); builder.addRangePartition(lower, upper); PartialRow upper = schema.newPartialRow(); upper.addString(0, "ghi"); builder.addRangePartition(lower, upper); PartialRow lower = schema.newPartialRow(); lower.addString(0, "z"); builder.addRangePartition(lower, schema.newPartialRow());
/** * Creates table options with non-covering range partitioning for a table with * the basic schema. Range partition key ranges fall between the following values: * * [ 0, 50) * [ 50, 100) * [200, 300) */ public static CreateTableOptions getBasicTableOptionsWithNonCoveredRange() { Schema schema = getBasicSchema(); CreateTableOptions option = new CreateTableOptions(); option.setRangePartitionColumns(ImmutableList.of("key")); PartialRow aLowerBound = schema.newPartialRow(); aLowerBound.addInt("key", 0); PartialRow aUpperBound = schema.newPartialRow(); aUpperBound.addInt("key", 100); option.addRangePartition(aLowerBound, aUpperBound); PartialRow bLowerBound = schema.newPartialRow(); bLowerBound.addInt("key", 200); PartialRow bUpperBound = schema.newPartialRow(); bUpperBound.addInt("key", 300); option.addRangePartition(bLowerBound, bUpperBound); PartialRow split = schema.newPartialRow(); split.addInt("key", 50); option.addSplitRow(split); return option; }
public void testScanTokensNonCoveringRangePartitions() throws Exception { Schema schema = createManyStringsSchema(); CreateTableOptions createOptions = new CreateTableOptions(); createOptions.addHashPartitions(ImmutableList.of("key"), 2); lower.addString("key", "a"); upper.addString("key", "f"); createOptions.addRangePartition(lower, upper); lower.addString("key", "h"); upper.addString("key", "z"); createOptions.addRangePartition(lower, upper); createOptions.addSplitRow(split);
private void createAndFillSchemasTable() throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(); ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING) .key(true).build(); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.setNumReplicas(1); // TODO config options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); try { for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); fillSchemaRow(insert.getRow(), schemaName); session.apply(insert); } } finally { session.close(); } }
CreateTableOptions createOptions = new CreateTableOptions(); createOptions.addHashPartitions(ImmutableList.of("key"), 8); createOptions.addSplitRow(splitRow);
@Before public void setUp() throws Exception { CreateTableOptions options = getBasicCreateTableOptions().setNumReplicas(1); table = harness.getClient().createTable(TABLE_NAME, getBasicSchema(), options); }
@Override public void updateSchema(VectorAccessible batch) throws IOException { BatchSchema schema = batch.getSchema(); int i = 0; try { if (!checkForTable(name)) { List<ColumnSchema> columns = new ArrayList<>(); for (MaterializedField f : schema) { columns.add(new ColumnSchema.ColumnSchemaBuilder(f.getName(), getType(f.getType())) .nullable(f.getType().getMode() == DataMode.OPTIONAL) .key(i == 0).build()); i++; } Schema kuduSchema = new Schema(columns); table = client.createTable(name, kuduSchema, new CreateTableOptions()); } } catch (Exception e) { throw new IOException(e); } }
/** * Add a set of hash partitions to the table. * * Each column must be a part of the table's primary key, and an individual * column may only appear in a single hash component. * * For each set of hash partitions added to the table, the total number of * table partitions is multiplied by the number of buckets. For example, if a * table is created with 3 split rows, and two hash partitions with 4 and 5 * buckets respectively, the total number of table partitions will be 80 * (4 range partitions * 4 hash buckets * 5 hash buckets). * * @param columns the columns to hash * @param buckets the number of buckets to hash into * @return this instance */ public CreateTableOptions addHashPartitions(List<String> columns, int buckets) { addHashPartitions(columns, buckets, 0); return this; }
/** * Add a range partition partition to the table with an inclusive lower bound * and an exclusive upper bound. * * If either row is empty, then that end of the range will be unbounded. If a * range column is missing a value, the logical minimum value for that column * type will be used as the default. * * Multiple range bounds may be added, but they must not overlap. All split * rows must fall in one of the range bounds. The lower bound must be less * than the upper bound. * * If not provided, the table's range will be unbounded. * * @param lower the inclusive lower bound * @param upper the exclusive upper bound * @return this instance */ public CreateTableOptions addRangePartition(PartialRow lower, PartialRow upper) { return addRangePartition(lower, upper, RangePartitionBound.INCLUSIVE_BOUND, RangePartitionBound.EXCLUSIVE_BOUND); }
private KuduTable createTableWithSplitsAndTest(String tableNamePrefix, int splitsCount) throws Exception { String newTableName = tableNamePrefix + "-" + splitsCount; CreateTableOptions builder = getBasicCreateTableOptions(); if (splitsCount != 0) { for (int i = 1; i <= splitsCount; i++) { PartialRow row = BASIC_SCHEMA.newPartialRow(); row.addInt(0, i); builder.addSplitRow(row); } } KuduTable table = client.createTable(newTableName, BASIC_SCHEMA, builder); List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); assertEquals(splitsCount + 1, tablets.size()); assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size()); for (LocatedTablet tablet : tablets) { assertEquals(3, tablet.getReplicas().size()); } return table; }
private static CreateTableOptions createTableOptions() { return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key")); }
@Test(timeout = 100000) public void testNonDefaultRangePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b")); PartialRow split = schema.newPartialRow(); split.addString("c", "3"); tableBuilder.addSplitRow(split); split = schema.newPartialRow(); split.addString("c", "3"); split.addString("b", "3"); tableBuilder.addSplitRow(split); testPartitionSchema(tableBuilder); }
Schema schema = new Schema(ImmutableList.of(a, b, c)); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.setRangePartitionColumns(new ArrayList<String>()); tableBuilder.addHashPartitions(ImmutableList.of("a"), 3); tableBuilder.addHashPartitions(ImmutableList.of("b"), 3); tableBuilder.addHashPartitions(ImmutableList.of("c"), 3);
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }