private static RangeBoundValue buildRangePartitionBound(KuduTable table, byte[] rangeKey) { if (rangeKey.length == 0) { return null; } else { Schema schema = table.getSchema(); PartitionSchema partitionSchema = table.getPartitionSchema(); PartitionSchema.RangeSchema rangeSchema = partitionSchema.getRangeSchema(); List<Integer> rangeColumns = rangeSchema.getColumns(); final int numColumns = rangeColumns.size(); PartialRow bound = KeyEncoderAccessor.decodeRangePartitionKey(schema, partitionSchema, rangeKey); ArrayList<Object> list = new ArrayList<>(); for (int i = 0; i < numColumns; i++) { Object obj = toValue(schema, bound, rangeColumns.get(i)); list.add(obj); } return new RangeBoundValue(list); } }
@Override public void startRecord() throws IOException { insert = table.newInsert(); setUp(insert.getRow()); }
@Override public void deleteRows(Block rowIds) { Schema schema = table.getSchema(); KuduSession session = clientSession.newSession(); session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND); try { try { for (int i = 0; i < rowIds.getPositionCount(); i++) { int len = rowIds.getSliceLength(i); Slice slice = rowIds.getSlice(i, 0, len); PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes()); Delete delete = table.newDelete(); RowHelper.copyPrimaryKey(schema, row, delete.getRow()); session.apply(delete); } } finally { session.close(); } } catch (KuduException e) { throw new RuntimeException(e); } }
operation = table.newDelete(); break; case INSERT: operation = table.newInsert(); break; case UPDATE: operation = table.newUpdate(); break; case UPSERT: operation = table.newUpsert(); break; default: if (kuduFieldNames == null) { kuduFieldNames = Lists.newArrayList(); for (ColumnSchema columnSchema : table.getSchema().getColumns()) { kuduFieldNames.add(columnSchema.getName()); ColumnSchema columnSchema = table.getSchema().getColumn(fieldName);
public Schema getTableSchema(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(this); return table.getSchema(); }
proto.setTableName(table.getName()); ColumnSchema columnSchema = table.getSchema().getColumn(columnName); Preconditions.checkArgument(columnSchema != null, "unknown column i%s", columnName); ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), columnSchema); ColumnSchema columnSchema = table.getSchema().getColumnByIndex(columnIdx); Preconditions.checkArgument(columnSchema != null, "unknown column index %s", columnIdx); ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), columnSchema); for (ColumnSchema column : table.getSchema().getColumns()) { ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), column); if (table.getAsyncClient().getLastPropagatedTimestamp() != AsyncKuduClient.NO_TIMESTAMP) { proto.setPropagatedTimestamp(client.getLastPropagatedTimestamp()); while (pruner.hasMorePartitionKeyRanges()) { Pair<byte[], byte[]> partitionRange = pruner.nextPartitionKeyRange(); List<LocatedTablet> newTablets = table.getTabletsLocations( partitionRange.getFirst().length == 0 ? null : partitionRange.getFirst(), partitionRange.getSecond().length == 0 ? null : partitionRange.getSecond(),
@Override public CompletableFuture<?> appendPage(Page page) { for (int position = 0; position < page.getPositionCount(); position++) { Upsert upsert = table.newUpsert(); PartialRow row = upsert.getRow(); int start = 0; if (generateUUID) { String id = String.format("%s-%08x", uuid, nextSubId++); row.addString(0, id); start = 1; } for (int channel = 0; channel < page.getChannelCount(); channel++) { appendColumn(row, page, position, channel, channel + start); } try { session.apply(upsert); } catch (KuduException e) { throw new RuntimeException(e); } } return NOT_BLOCKED; }
@Override public Status delete(String table, String key) { Delete delete = this.kuduTable.newDelete(); PartialRow row = delete.getRow(); row.addString(KEY, key); apply(delete); return Status.OK; }
assertEquals(defaultInt, kuduTable.getSchema().getColumnByIndex(0).getDefaultValue()); assertEquals(defaultString, kuduTable.getSchema().getColumnByIndex(columns.size() - 2).getDefaultValue()); assertEquals(true, kuduTable.getSchema().getColumnByIndex(columns.size() - 1).getDefaultValue()); assertTrue(kuduTable.getSchema().hasColumnIds()); assertNull(kuduTableWithoutDefaults.getSchema().getColumnByIndex(0).getDefaultValue()); createTableWithSplitsAndTest(splitTablePrefix, 3); createTableWithSplitsAndTest(splitTablePrefix, 10); List<LocatedTablet>tablets = table.getTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP); assertEquals(9, tablets.size()); assertEquals(9, table.asyncGetTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP).join().size()); tablets = table.getTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP); assertEquals(9, tablets.size()); assertEquals(9, table.asyncGetTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP).join().size()); tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP); assertEquals(4, tablets.size()); assertEquals(4, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP).join().size()); tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP); assertEquals(9, tablets.size()); assertEquals(9, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP).join().size()); tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP);
private static List<RangePartition> getRangePartitionList(KuduTable table, long deadline) { List<RangePartition> rangePartitions = new ArrayList<>(); if (!table.getPartitionSchema().getRangeSchema().getColumns().isEmpty()) { try { Iterator var4 = table.getTabletsLocations(deadline).iterator(); while (var4.hasNext()) { LocatedTablet tablet = (LocatedTablet) var4.next(); Partition partition = tablet.getPartition(); if (Iterators.all(partition.getHashBuckets().iterator(), Predicates.equalTo(0))) { RangePartition rangePartition = buildRangePartition(table, partition); rangePartitions.add(rangePartition); } } } catch (Exception e) { throw new RuntimeException(e); } } return rangePartitions; }
@Test(timeout = 100000) public void testBatchWithSameRow() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end, // while also clearing the cache between each batch half the time. The delete is added here // so that a misplaced update would fail if it happens later than its delete. for (int i = 0; i < 25; i++) { session.apply(createInsert(table, i)); for (int j = 0; j < 50; j++) { Update update = table.newUpdate(); PartialRow row = update.getRow(); row.addInt(basicSchema.getColumnByIndex(0).getName(), i); row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000); session.apply(update); } Delete del = table.newDelete(); PartialRow row = del.getRow(); row.addInt(basicSchema.getColumnByIndex(0).getName(), i); session.apply(del); session.flush(); if (i % 2 == 0) { asyncClient.emptyTabletsCacheForTable(table.getTableId()); } } assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build())); }
private void init() { String tableName = kuduScanSpec.getTableName(); Collection<DrillbitEndpoint> endpoints = kuduStoragePlugin.getContext().getBits(); Map<String,DrillbitEndpoint> endpointMap = Maps.newHashMap(); for (DrillbitEndpoint endpoint : endpoints) { endpointMap.put(endpoint.getAddress(), endpoint); } try { List<LocatedTablet> locations = kuduStoragePlugin.getClient().openTable(tableName).getTabletsLocations(10000); for (LocatedTablet tablet : locations) { KuduWork work = new KuduWork(tablet.getPartition().getPartitionKeyStart(), tablet.getPartition().getPartitionKeyEnd()); for (Replica replica : tablet.getReplicas()) { String host = replica.getRpcHost(); DrillbitEndpoint ep = endpointMap.get(host); if (ep != null) { work.getByteMap().add(ep, DEFAULT_TABLET_SIZE); } } kuduWorkList.add(work); } } catch (Exception e) { throw new RuntimeException(e); } }
/** * Helper method to easily kill a tablet server that serves the given table's only tablet's * leader. The currently running test case will be failed if there's more than one tablet, * if the tablet has no leader after some retries, or if the tablet server was already killed. * * This method is thread-safe. * @param table a KuduTable which will get its single tablet's leader killed. * @throws Exception */ public void killTabletLeader(KuduTable table) throws Exception { List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); if (tablets.isEmpty() || tablets.size() > 1) { fail("Currently only support killing leaders for tables containing 1 tablet, table " + table.getName() + " has " + tablets.size()); } LocatedTablet tablet = tablets.get(0); if (tablet.getReplicas().size() == 1) { fail("Table " + table.getName() + " only has 1 tablet, please enable replication"); } HostAndPort hp = findLeaderTabletServer(tablet); miniCluster.killTabletServer(hp); }
@Override public Status update(String table, String key, Map<String, ByteIterator> values) { Update update = this.kuduTable.newUpdate(); PartialRow row = update.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { String columnName = schema.getColumnByIndex(i).getName(); ByteIterator b = values.get(columnName); if (b != null) { row.addStringUtf8(columnName, b.toArray()); } } apply(update); return Status.OK; }
@Override public String toString() { return "get tablet locations from the master for table " + table.getName(); } }
public static Map<String, Object> toMap(KuduTable table) { Map<String, Object> properties = new HashMap<>(); LinkedHashMap<String, ColumnDesign> columns = getColumns(table); PartitionDesign partitionDesign = getPartitionDesign(table); List<RangePartition> rangePartitionList = getRangePartitionList(table, DEFAULT_DEADLINE); try { if (partitionDesign.getHash() != null) { List<HashPartitionDefinition> list = partitionDesign.getHash(); if (!list.isEmpty()) { properties.put(PARTITION_BY_HASH_COLUMNS, list.get(0).getColumns()); properties.put(PARTITION_BY_HASH_BUCKETS, list.get(0).getBuckets()); } if (list.size() >= 2) { properties.put(PARTITION_BY_HASH_COLUMNS_2, list.get(1).getColumns()); properties.put(PARTITION_BY_HASH_BUCKETS_2, list.get(1).getBuckets()); } } if (partitionDesign.getRange() != null) { properties.put(PARTITION_BY_RANGE_COLUMNS, partitionDesign.getRange().getColumns()); } String partitionRangesValue = mapper.writeValueAsString(rangePartitionList); properties.put(RANGE_PARTITIONS, partitionRangesValue); properties.put(NUM_REPLICAS, table.getNumReplicas()); return properties; } catch (IOException e) { throw new RuntimeException(e); } }