builder.setRangePartitionColumns(Arrays.asList("key")); for (int i = 1; i < tablets; i++) { PartialRow splitRow = schema.newPartialRow(); splitRow.addInt("key", i*1000); builder.addSplitRow(splitRow);
/** * Package-private constructor. Subclasses need to be instantiated via AsyncKuduSession * @param table table with the schema to use for this operation */ Operation(KuduTable table) { super(table); this.row = table.getSchema().newPartialRow(); }
/** * Decodes a primary key into a row * * @param schema the table schema * @param key the encoded key * @return the decoded primary key as a row */ public static PartialRow decodePrimaryKey(Schema schema, byte[] key) { PartialRow row = schema.newPartialRow(); ByteBuffer buf = ByteBuffer.wrap(key); buf.order(ByteOrder.BIG_ENDIAN); for (int idx = 0; idx < schema.getPrimaryKeyColumnCount(); idx++) { decodeColumn(buf, row, idx, idx + 1 == schema.getPrimaryKeyColumnCount()); } if (buf.hasRemaining()) { throw new IllegalArgumentException("Unable to decode all primary key bytes"); } return row; }
/** * Decodes a range partition key into a partial row. * * @param schema the schema of the table * @param partitionSchema the partition schema of the table * @param buf the encoded range partition key * @return the decoded range key */ private static PartialRow decodeRangePartitionKey(Schema schema, PartitionSchema partitionSchema, ByteBuffer buf) { PartialRow row = schema.newPartialRow(); Iterator<Integer> rangeIds = partitionSchema.getRangeSchema().getColumns().iterator(); while (rangeIds.hasNext()) { int idx = schema.getColumnIndex(rangeIds.next()); if (buf.hasRemaining()) { decodeColumn(buf, row, idx, !rangeIds.hasNext()); } else { row.setMin(idx); } } if (buf.hasRemaining()) { throw new IllegalArgumentException("Unable to decode all partition key bytes"); } return row; }
/** * Gets the decoded start range key. * @return the decoded start range key */ public PartialRow getDecodedRangeKeyStart(KuduTable table) { Schema schema = table.getSchema(); if (rangeKeyStart.length == 0) { return schema.newPartialRow(); } else { PartitionSchema partitionSchema = table.getPartitionSchema(); return KeyEncoder.decodeRangePartitionKey(schema, partitionSchema, rangeKeyStart); } }
/** * Gets the decoded end range key. * @return the decoded end range key */ public PartialRow getDecodedRangeKeyEnd(KuduTable table) { Schema schema = table.getSchema(); if (rangeKeyEnd.length == 0) { return schema.newPartialRow(); } else { PartitionSchema partitionSchema = table.getPartitionSchema(); return KeyEncoder.decodeRangePartitionKey(schema, partitionSchema, rangeKeyEnd); } }
public static CreateTableOptions getCreateTableOptions(Schema schema, int numReplicas, int rangePartitions, int hashPartitions) { Preconditions.checkArgument(rangePartitions > 0); Preconditions.checkArgument(hashPartitions > 0); CreateTableOptions options = new CreateTableOptions().setNumReplicas(numReplicas); if (rangePartitions > 1) { options.setRangePartitionColumns(ImmutableList.of(COLUMN_KEY_ONE)); BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); BigInteger step = max.multiply(BigInteger.valueOf(2)) .divide(BigInteger.valueOf(rangePartitions)); PartialRow splitRow = schema.newPartialRow(); for (int i = 1; i < rangePartitions; i++) { long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue(); splitRow.addLong(COLUMN_KEY_ONE_IDX, key); options.addSplitRow(splitRow); } } else { options.setRangePartitionColumns(Collections.<String>emptyList()); } if (hashPartitions > 1) { options.addHashPartitions(ImmutableList.of(COLUMN_KEY_ONE), hashPartitions); } return options; }
private static void configureScannerForRandomRead(AbstractKuduScannerBuilder builder, KuduTable table, long keyOne, long keyTwo) { PartialRow lowerBound = table.getSchema().newPartialRow(); lowerBound.addLong(0, keyOne); lowerBound.addLong(1, keyTwo); builder.lowerBound(lowerBound); PartialRow upperBound = table.getSchema().newPartialRow(); // Adding 1 since we want a single row, and the upper bound is exclusive. upperBound.addLong(0, keyOne + 1); upperBound.addLong(1, keyTwo + 1); builder.exclusiveUpperBound(upperBound); }
public byte[] getKeyInBytes(int i) { PartialRow row = BASIC_SCHEMA.newPartialRow(); row.addInt(0, i); return row.encodePrimaryKey(); }
@Test(timeout = 100000) public void testSimplePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("a", "b", "c")); PartialRow split = schema.newPartialRow(); split.addString("c", "3"); tableBuilder.addSplitRow(split); split = schema.newPartialRow(); split.addString("c", "3"); split.addString("b", "3"); tableBuilder.addSplitRow(split); testPartitionSchema(tableBuilder); }
@Test(timeout = 100000) public void testHashBucketedAndRangePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.addHashPartitions(ImmutableList.of("a"), 3); tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42); tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b")); PartialRow split = schema.newPartialRow(); split.addString("c", "3"); tableBuilder.addSplitRow(split); split = schema.newPartialRow(); split.addString("c", "3"); split.addString("b", "3"); tableBuilder.addSplitRow(split); testPartitionSchema(tableBuilder); }
@Test(timeout = 100000) public void testNonDefaultRangePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b")); PartialRow split = schema.newPartialRow(); split.addString("c", "3"); tableBuilder.addSplitRow(split); split = schema.newPartialRow(); split.addString("c", "3"); split.addString("b", "3"); tableBuilder.addSplitRow(split); testPartitionSchema(tableBuilder); }
private AsyncKuduScanner getScanner(int start, int exclusiveEnd, List<String> columnNames) { PartialRow lowerBound = schema.newPartialRow(); lowerBound.addInt(schema.getColumnByIndex(0).getName(), start); PartialRow upperBound = schema.newPartialRow(); upperBound.addInt(schema.getColumnByIndex(0).getName(), exclusiveEnd); return client.newScannerBuilder(table) .lowerBound(lowerBound) .exclusiveUpperBound(upperBound) .setProjectedColumnNames(columnNames) .build(); }
@Test(timeout = 100000) public void testNonCoveredRangePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c")); // Create a non covered range between (3, 5, 6) and (4, 0, 0) PartialRow lowerBoundA = schema.newPartialRow(); lowerBoundA.addString("a", "0"); lowerBoundA.addString("b", "0"); lowerBoundA.addString("c", "0"); PartialRow upperBoundA = schema.newPartialRow(); upperBoundA.addString("a", "3"); upperBoundA.addString("b", "5"); upperBoundA.addString("b", "6"); tableBuilder.addRangePartition(lowerBoundA, upperBoundA); PartialRow lowerBoundB = schema.newPartialRow(); lowerBoundB.addString("a", "4"); lowerBoundB.addString("b", "0"); lowerBoundB.addString("c", "0"); PartialRow upperBoundB = schema.newPartialRow(); upperBoundB.addString("a", "5"); upperBoundB.addString("b", "5"); upperBoundB.addString("b", "6"); tableBuilder.addRangePartition(lowerBoundB, upperBoundB); testPartitionSchema(tableBuilder); }
private KuduTable createTableWithSplitsAndTest(String tableNamePrefix, int splitsCount) throws Exception { String newTableName = tableNamePrefix + "-" + splitsCount; CreateTableOptions builder = getBasicCreateTableOptions(); if (splitsCount != 0) { for (int i = 1; i <= splitsCount; i++) { PartialRow row = BASIC_SCHEMA.newPartialRow(); row.addInt(0, i); builder.addSplitRow(row); } } KuduTable table = client.createTable(newTableName, BASIC_SCHEMA, builder); List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); assertEquals(splitsCount + 1, tablets.size()); assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size()); for (LocatedTablet tablet : tablets) { assertEquals(3, tablet.getReplicas().size()); } return table; }
private AsyncKuduScanner getScanner(String lowerBoundKeyOne, String lowerBoundKeyTwo, String exclusiveUpperBoundKeyOne, String exclusiveUpperBoundKeyTwo, ColumnRangePredicate predicate) { AsyncKuduScanner.AsyncKuduScannerBuilder builder = asyncClient.newScannerBuilder(table); if (lowerBoundKeyOne != null) { PartialRow lowerBoundRow = schema.newPartialRow(); lowerBoundRow.addString(0, lowerBoundKeyOne); lowerBoundRow.addString(1, lowerBoundKeyTwo); builder.lowerBound(lowerBoundRow); } if (exclusiveUpperBoundKeyOne != null) { PartialRow upperBoundRow = schema.newPartialRow(); upperBoundRow.addString(0, exclusiveUpperBoundKeyOne); upperBoundRow.addString(1, exclusiveUpperBoundKeyTwo); builder.exclusiveUpperBound(upperBoundRow); } if (predicate != null) { builder.addColumnRangePredicate(predicate); } return builder.build(); }
private PartialRow getPartialRowWithAllTypes() { Schema schema = getSchemaWithAllTypes(); // Ensure we aren't missing any types assertEquals(13, schema.getColumnCount()); PartialRow row = schema.newPartialRow(); row.addByte("int8", (byte) 42); row.addShort("int16", (short) 43); row.addInt("int32", 44); row.addLong("int64", 45); row.addTimestamp("timestamp", new Timestamp(1234567890)); row.addBoolean("bool", true); row.addFloat("float", 52.35F); row.addDouble("double", 53.35); row.addString("string", "fun with ütf\0"); row.addBinary("binary-array", new byte[] { 0, 1, 2, 3, 4 }); ByteBuffer binaryBuffer = ByteBuffer.wrap(new byte[] { 5, 6, 7, 8, 9 }); row.addBinary("binary-bytebuffer", binaryBuffer); row.setNull("null"); row.addDecimal("decimal", BigDecimal.valueOf(12345, 3)); return row; }
@Test public void testToString() { Schema schema = getSchemaWithAllTypes(); PartialRow row = schema.newPartialRow(); assertEquals("()", row.toString()); row.addInt("int32", 42); row.addByte("int8", (byte) 42); assertEquals("(int8 int8=42, int32 int32=42)", row.toString()); row.addString("string", "fun with ütf\0"); assertEquals("(int8 int8=42, int32 int32=42, string string=\"fun with ütf\\0\")", row.toString()); ByteBuffer binary = ByteBuffer.wrap(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); binary.position(2); binary.limit(5); row.addBinary("binary-bytebuffer", binary); assertEquals("(int8 int8=42, int32 int32=42, string string=\"fun with ütf\\0\", " + "binary binary-bytebuffer=[2, 3, 4])", row.toString()); row.addDouble("double", 52.35); assertEquals("(int8 int8=42, int32 int32=42, double double=52.35, " + "string string=\"fun with ütf\\0\", binary binary-bytebuffer=[2, 3, 4])", row.toString()); row.addDecimal("decimal", BigDecimal.valueOf(12345, 3)); assertEquals("(int8 int8=42, int32 int32=42, double double=52.35, " + "string string=\"fun with ütf\\0\", binary binary-bytebuffer=[2, 3, 4], " + "decimal(5, 3) decimal=12.345)", row.toString()); }
@Test public void testGetUnsetColumn() { Schema schema = getSchemaWithAllTypes(); PartialRow partialRow = schema.newPartialRow(); for (ColumnSchema columnSchema : schema.getColumns()) { assertFalse(partialRow.isSet("null")); assertFalse(partialRow.isNull("null")); try { callGetByName(partialRow, columnSchema.getName(), columnSchema.getType()); fail("Expected IllegalArgumentException for type: " + columnSchema.getType()); } catch (IllegalArgumentException ex) { // This is the expected exception. } } }
public static KuduTable createFourTabletsTableWithNineRows(AsyncKuduClient client, String tableName, final long timeoutMs) throws Exception { final int[] KEYS = new int[] { 10, 20, 30 }; final Schema basicSchema = getBasicSchema(); CreateTableOptions builder = getBasicCreateTableOptions(); for (int i : KEYS) { PartialRow splitRow = basicSchema.newPartialRow(); splitRow.addInt(0, i); builder.addSplitRow(splitRow); } KuduTable table = client.syncClient().createTable(tableName, basicSchema, builder); AsyncKuduSession session = client.newSession(); // create a table with on empty tablet and 3 tablets of 3 rows each for (int key1 : KEYS) { for (int key2 = 1; key2 <= 3; key2++) { Insert insert = table.newInsert(); PartialRow row = insert.getRow(); row.addInt(0, key1 + key2); row.addInt(1, key1); row.addInt(2, key2); row.addString(3, "a string"); row.addBoolean(4, true); session.apply(insert).join(timeoutMs); } } session.close().join(timeoutMs); return table; }