private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(clientSession); Schema schema = table.getSchema(); List<ColumnMetadata> columnsMetaList = schema.getColumns().stream() .filter(column -> !column.isKey() || !column.getName().equals(KuduColumnHandle.ROW_ID)) .map(this::getColumnMetadata) .collect(toImmutableList()); Map<String, Object> properties = clientSession.getTableProperties(tableHandle); return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties); }
private Schema buildSchema(List<ColumnMetadata> columns, Map<String, Object> tableProperties) { List<ColumnSchema> kuduColumns = columns.stream() .map(this::toColumnSchema) .collect(ImmutableList.toImmutableList()); return new Schema(kuduColumns); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle; Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int ordinal = 0; ordinal < schema.getColumnCount(); ordinal++) { ColumnSchema col = schema.getColumnByIndex(ordinal); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, ordinal, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
public static PartitionDesign getPartitionDesign(KuduTable table) { Schema schema = table.getSchema(); PartitionDesign partitionDesign = new PartitionDesign(); PartitionSchema partitionSchema = table.getPartitionSchema(); List<HashPartitionDefinition> hashPartitions = partitionSchema.getHashBucketSchemas().stream() .map(hashBucketSchema -> { HashPartitionDefinition hash = new HashPartitionDefinition(); List<String> cols = hashBucketSchema.getColumnIds().stream() .map(idx -> schema.getColumnByIndex(idx).getName()).collect(toImmutableList()); hash.setColumns(cols); hash.setBuckets(hashBucketSchema.getNumBuckets()); return hash; }).collect(toImmutableList()); partitionDesign.setHash(hashPartitions); List<Integer> rangeColumns = partitionSchema.getRangeSchema().getColumns(); if (!rangeColumns.isEmpty()) { RangePartitionDefinition definition = new RangePartitionDefinition(); definition.setColumns(rangeColumns.stream() .map(i -> schema.getColumns().get(i).getName()) .collect(ImmutableList.toImmutableList())); partitionDesign.setRange(definition); } return partitionDesign; }
public static void copyPrimaryKey(Schema schema, RowResult from, PartialRow to) for (int i = 0; i < schema.getPrimaryKeyColumnCount(); i++) { switch (schema.getColumnByIndex(i).getType()) { case STRING: to.addStringUtf8(i, from.getString(i).getBytes(StandardCharsets.UTF_8)); break; default: throw new IllegalStateException("Unknown type " + schema.getColumnByIndex(i).getType() + " for column " + schema.getColumnByIndex(i).getName());
/** * Test creating and deleting a table through a KuduClient. */ @Test(timeout = 100000) public void testCreateDeleteTable() throws Exception { // Check that we can create a table. client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); assertFalse(client.getTablesList().getTablesList().isEmpty()); assertTrue(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can delete it. client.deleteTable(TABLE_NAME); assertFalse(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can re-recreate it, with a different schema. List<ColumnSchema> columns = new ArrayList<>(basicSchema.getColumns()); columns.add(new ColumnSchema.ColumnSchemaBuilder("one more", Type.STRING).build()); Schema newSchema = new Schema(columns); client.createTable(TABLE_NAME, newSchema, getBasicCreateTableOptions()); // Check that we can open a table and see that it has the new schema. KuduTable table = client.openTable(TABLE_NAME); assertEquals(newSchema.getColumnCount(), table.getSchema().getColumnCount()); assertTrue(table.getPartitionSchema().isSimpleRangePartitioning()); // Check that the block size parameter we specified in the schema is respected. assertEquals(4096, newSchema.getColumn("column3_s").getDesiredBlockSize()); assertEquals(ColumnSchema.Encoding.DICT_ENCODING, newSchema.getColumn("column3_s").getEncoding()); assertEquals(ColumnSchema.CompressionAlgorithm.LZ4, newSchema.getColumn("column3_s").getCompressionAlgorithm()); }
public static PartialRow toRangeBoundToPartialRow(Schema schema, RangePartitionDefinition definition, RangeBoundValue boundValue) { PartialRow partialRow = new PartialRow(schema); if (boundValue != null) { List<Integer> rangeColumns = definition.getColumns().stream() .map(schema::getColumnIndex).collect(toImmutableList()); if (rangeColumns.size() != boundValue.getValues().size()) { throw new IllegalStateException("Expected " + rangeColumns.size() + " range columns, but got " + boundValue.getValues().size()); } for (int i = 0; i < rangeColumns.size(); i++) { Object obj = boundValue.getValues().get(i); int idx = rangeColumns.get(i); ColumnSchema columnSchema = schema.getColumnByIndex(idx); setColumnValue(partialRow, idx, obj, columnSchema.getType(), columnSchema.getName()); } } return partialRow; }
/** * Utility method to convert a Schema to its wire format. * @param schema Schema to convert * @return a list of ColumnSchemaPB */ public static List<Common.ColumnSchemaPB> schemaToListPb(Schema schema) { ArrayList<Common.ColumnSchemaPB> columns = new ArrayList<Common.ColumnSchemaPB>(schema.getColumnCount()); Common.ColumnSchemaPB.Builder schemaBuilder = Common.ColumnSchemaPB.newBuilder(); for (ColumnSchema col : schema.getColumns()) { columns.add(columnToPb(schemaBuilder, col)); schemaBuilder.clear(); } return columns; }
/** * Checks for column. * * @param schema * the schema * @param columnName * the column name * @return true, if successful */ public static boolean hasColumn(Schema schema, String columnName) { try { schema.getColumn(columnName); return true; } catch (IllegalArgumentException e) { return false; } }
private int getColumnIndex(Schema columns, String colName) { try { return columns.getColumnIndex(colName); } catch (Exception ex) { return -1; } } }
/** * @throws IndexOutOfBoundsException if the column doesn't exist */ private void checkValidColumn(int columnIndex) { if (columnIndex >= schema.getColumnCount()) { throw new IndexOutOfBoundsException("Requested column is out of range, " + columnIndex + " out of " + schema.getColumnCount()); } }
columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build()); Schema schema = new Schema(columns); builder.setRangePartitionColumns(Arrays.asList("key")); for (int i = 1; i < tablets; i++) { PartialRow splitRow = schema.newPartialRow(); splitRow.addInt("key", i*1000); builder.addSplitRow(splitRow);
@Test public void testGetUnsetColumn() { Schema schema = getSchemaWithAllTypes(); PartialRow partialRow = schema.newPartialRow(); for (ColumnSchema columnSchema : schema.getColumns()) { assertFalse(partialRow.isSet("null")); assertFalse(partialRow.isNull("null")); try { callGetByName(partialRow, columnSchema.getName(), columnSchema.getType()); fail("Expected IllegalArgumentException for type: " + columnSchema.getType()); } catch (IllegalArgumentException ex) { // This is the expected exception. } } }
private AsyncKuduScanner getScanner(int start, int exclusiveEnd, List<String> columnNames) { PartialRow lowerBound = schema.newPartialRow(); lowerBound.addInt(schema.getColumnByIndex(0).getName(), start); PartialRow upperBound = schema.newPartialRow(); upperBound.addInt(schema.getColumnByIndex(0).getName(), exclusiveEnd); return client.newScannerBuilder(table) .lowerBound(lowerBound) .exclusiveUpperBound(upperBound) .setProjectedColumnNames(columnNames) .build(); }
private PartialRow getPartialRowWithAllTypes() { Schema schema = getSchemaWithAllTypes(); // Ensure we aren't missing any types assertEquals(13, schema.getColumnCount()); PartialRow row = schema.newPartialRow(); row.addByte("int8", (byte) 42); row.addShort("int16", (short) 43); row.addInt("int32", 44); row.addLong("int64", 45); row.addTimestamp("timestamp", new Timestamp(1234567890)); row.addBoolean("bool", true); row.addFloat("float", 52.35F); row.addDouble("double", 53.35); row.addString("string", "fun with ütf\0"); row.addBinary("binary-array", new byte[] { 0, 1, 2, 3, 4 }); ByteBuffer binaryBuffer = ByteBuffer.wrap(new byte[] { 5, 6, 7, 8, 9 }); row.addBinary("binary-bytebuffer", binaryBuffer); row.setNull("null"); row.addDecimal("decimal", BigDecimal.valueOf(12345, 3)); return row; }
final int primaryKeyColumnCount = table.getSchema().getPrimaryKeyColumnCount(); KuduScanToken.KuduScanTokenBuilder builder = client.newScanTokenBuilder(table);
/** * Package-private constructor. Subclasses need to be instantiated via AsyncKuduSession * @param table table with the schema to use for this operation */ Operation(KuduTable table) { super(table); this.row = table.getSchema().newPartialRow(); }
/** * This is not a stable API, prefer using {@link Schema#newPartialRow()} * to create a new partial row. * @param schema the schema to use for this row */ public PartialRow(Schema schema) { this.schema = schema; this.columnsBitSet = new BitSet(this.schema.getColumnCount()); this.nullsBitSet = schema.hasNullableColumns() ? new BitSet(this.schema.getColumnCount()) : null; this.rowAlloc = new byte[schema.getRowSize()]; // Pre-fill the array with nulls. We'll only replace cells that have varlen values. this.varLengthData = Arrays.asList(new ByteBuffer[this.schema.getColumnCount()]); }
public KuduRow(Object object, Schema schema) { super(validFields(object)); for (Class<?> c = object.getClass(); c != null; c = c.getSuperclass()) { basicValidation(c.getDeclaredFields()) .filter(field -> schema.getColumn(field.getName()) != null) .forEach(cField -> { try { cField.setAccessible(true); setField(schema.getColumnIndex(cField.getName()), cField.getName(), cField.get(object)); } catch (IllegalAccessException e) { String error = String.format("Cannot get value for %s", cField.getName()); throw new IllegalArgumentException(error, e); } }); } }
/** * Creates a new partition schema from the range and hash bucket schemas. * * @param rangeSchema the range schema * @param hashBucketSchemas the hash bucket schemas * @param schema the table schema */ PartitionSchema(RangeSchema rangeSchema, List<HashBucketSchema> hashBucketSchemas, Schema schema) { this.rangeSchema = rangeSchema; this.hashBucketSchemas = hashBucketSchemas; boolean isSimple = hashBucketSchemas.isEmpty() && rangeSchema.columns.size() == schema.getPrimaryKeyColumnCount(); if (isSimple) { int i = 0; for (Integer id : rangeSchema.columns) { if (schema.getColumnIndex(id) != i++) { isSimple = false; break; } } } this.isSimple = isSimple; }