@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle; Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int ordinal = 0; ordinal < schema.getColumnCount(); ordinal++) { ColumnSchema col = schema.getColumnByIndex(ordinal); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, ordinal, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
@Override public Status insert(String table, String key, Map<String, ByteIterator> values) { Insert insert = this.kuduTable.newInsert(); PartialRow row = insert.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { row.addStringUtf8(i, values.get(schema.getColumnByIndex(i).getName()).toArray()); } apply(insert); return Status.OK; }
@Override public Status update(String table, String key, Map<String, ByteIterator> values) { Update update = this.kuduTable.newUpdate(); PartialRow row = update.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { String columnName = schema.getColumnByIndex(i).getName(); ByteIterator b = values.get(columnName); if (b != null) { row.addStringUtf8(columnName, b.toArray()); } } apply(update); return Status.OK; }
private void initCols(Schema schema) throws SchemaChangeException { ImmutableList.Builder<ProjectedColumnInfo> pciBuilder = ImmutableList.builder(); for (int i = 0; i < schema.getColumnCount(); i++) { ColumnSchema col = schema.getColumnByIndex(i);
/** * @throws IndexOutOfBoundsException if the column doesn't exist */ private void checkValidColumn(int columnIndex) { if (columnIndex >= schema.getColumnCount()) { throw new IndexOutOfBoundsException("Requested column is out of range, " + columnIndex + " out of " + schema.getColumnCount()); } }
/** * This is not a stable API, prefer using {@link Schema#newPartialRow()} * to create a new partial row. * @param schema the schema to use for this row */ public PartialRow(Schema schema) { this.schema = schema; this.columnsBitSet = new BitSet(this.schema.getColumnCount()); this.nullsBitSet = schema.hasNullableColumns() ? new BitSet(this.schema.getColumnCount()) : null; this.rowAlloc = new byte[schema.getRowSize()]; // Pre-fill the array with nulls. We'll only replace cells that have varlen values. this.varLengthData = Arrays.asList(new ByteBuffer[this.schema.getColumnCount()]); }
void advancePointerTo(int rowIndex) { this.index = rowIndex; this.offset = this.rowSize * this.index; if (schema.hasNullableColumns() && this.index != INDEX_RESET_LOCATION) { this.nullsBitSet = Bytes.toBitSet( this.rowData.getRawArray(), this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(schema.getColumnCount()), schema.getColumnCount()); } }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle; Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int ordinal = 0; ordinal < schema.getColumnCount(); ordinal++) { ColumnSchema col = schema.getColumnByIndex(ordinal); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, ordinal, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
/** * Utility method to convert a Schema to its wire format. * @param schema Schema to convert * @return a list of ColumnSchemaPB */ public static List<Common.ColumnSchemaPB> schemaToListPb(Schema schema) { ArrayList<Common.ColumnSchemaPB> columns = new ArrayList<Common.ColumnSchemaPB>(schema.getColumnCount()); Common.ColumnSchemaPB.Builder schemaBuilder = Common.ColumnSchemaPB.newBuilder(); for (ColumnSchema col : schema.getColumns()) { columns.add(columnToPb(schemaBuilder, col)); schemaBuilder.clear(); } return columns; }
/** * Initializes the state of the encoder based on the schema and number of operations to encode. * * @param schema the schema of the table which the operations belong to. * @param numOperations the number of operations. */ private void init(Schema schema, int numOperations) { this.schema = schema; // Set up the encoded data. // Estimate a maximum size for the data. This is conservative, but avoids // having to loop through all the operations twice. final int columnBitSetSize = Bytes.getBitSetSize(schema.getColumnCount()); int sizePerRow = 1 /* for the op type */ + schema.getRowSize() + columnBitSetSize; if (schema.hasNullableColumns()) { // nullsBitSet is the same size as the columnBitSet sizePerRow += columnBitSetSize; } // TODO: would be more efficient to use a buffer which "chains" smaller allocations // instead of a doubling buffer like BAOS. this.rows = ByteBuffer.allocate(sizePerRow * numOperations) .order(ByteOrder.LITTLE_ENDIAN); this.indirect = new ArrayList<>(schema.getVarLengthColumnCount() * numOperations); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle); Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int i = 0; i < schema.getColumnCount(); i++) { ColumnSchema col = schema.getColumnByIndex(i); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, i, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
/** {@inheritDoc} */ @Override public String toString() { int numCols = schema.getColumnCount(); StringBuilder sb = new StringBuilder(); sb.append('('); boolean first = true; for (int idx = 0; idx < numCols; ++idx) { if (!columnsBitSet.get(idx)) { continue; } if (first) { first = false; } else { sb.append(", "); } ColumnSchema col = schema.getColumnByIndex(idx); sb.append(col.getType().getName()); if (col.getTypeAttributes() != null) { sb.append(col.getTypeAttributes().toStringForType(col.getType())); } sb.append(' '); sb.append(col.getName()); sb.append('='); appendCellValueDebugString(idx, sb); } sb.append(')'); return sb.toString(); }
private void encodeRow(PartialRow row, ChangeType type) { rows.put(type.toEncodedByte()); rows.put(Bytes.fromBitSet(row.getColumnsBitSet(), schema.getColumnCount())); if (schema.hasNullableColumns()) { rows.put(Bytes.fromBitSet(row.getNullsBitSet(), schema.getColumnCount())); } int colIdx = 0; byte[] rowData = row.getRowAlloc(); int currentRowOffset = 0; for (ColumnSchema col : row.getSchema().getColumns()) { // Keys should always be specified, maybe check? if (row.isSet(colIdx) && !row.isSetToNull(colIdx)) { if (col.getType() == Type.STRING || col.getType() == Type.BINARY) { ByteBuffer varLengthData = row.getVarLengthData().get(colIdx); varLengthData.reset(); rows.putLong(indirectWrittenBytes); int bbSize = varLengthData.remaining(); rows.putLong(bbSize); indirect.add(varLengthData); indirectWrittenBytes += bbSize; } else { // This is for cols other than strings rows.put(rowData, currentRowOffset, col.getTypeSize()); } } currentRowOffset += col.getTypeSize(); colIdx++; } }
this.rowData = rowData; this.indirectData = indirectData; int columnOffsetsSize = schema.getColumnCount(); if (schema.hasNullableColumns()) { columnOffsetsSize++;
@Override @SuppressWarnings("AssertionFailureIgnored") public void run() { final String tableName = "TestAuthnTokenReacquire-table-" + threadIdx; try { ListTabletServersResponse response = client.listTabletServers(); assertNotNull(response); dropConnectionsAndExpireToken(); ListTablesResponse tableList = client.getTablesList(tableName); assertNotNull(tableList); assertTrue(tableList.getTablesList().isEmpty()); dropConnectionsAndExpireToken(); client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); dropConnectionsAndExpireToken(); KuduTable table = client.openTable(tableName); assertEquals(basicSchema.getColumnCount(), table.getSchema().getColumnCount()); dropConnectionsAndExpireToken(); client.deleteTable(tableName); assertFalse(client.tableExists(tableName)); } catch (Throwable e) { //noinspection ThrowableResultOfMethodCallIgnored exceptions.put(threadIdx, e); } } });
/** * Insert rows into the provided table. The table's columns must be ints, and * must have a primary key in the first column. * @param table the table * @param start the inclusive start key * @param end the exclusive end key */ private void insertRows(KuduTable table, int start, int end) throws KuduException { KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (int i = start; i < end; i++) { Insert insert = table.newInsert(); for (int idx = 0; idx < table.getSchema().getColumnCount(); idx++) { insert.getRow().addInt(idx, i); } session.apply(insert); } session.flush(); RowError[] rowErrors = session.getPendingErrors().getRowErrors(); assertEquals(String.format("row errors: %s", Arrays.toString(rowErrors)), 0, rowErrors.length); }
private PartialRow getPartialRowWithAllTypes() { Schema schema = getSchemaWithAllTypes(); // Ensure we aren't missing any types assertEquals(13, schema.getColumnCount()); PartialRow row = schema.newPartialRow(); row.addByte("int8", (byte) 42); row.addShort("int16", (short) 43); row.addInt("int32", 44); row.addLong("int64", 45); row.addTimestamp("timestamp", new Timestamp(1234567890)); row.addBoolean("bool", true); row.addFloat("float", 52.35F); row.addDouble("double", 53.35); row.addString("string", "fun with ütf\0"); row.addBinary("binary-array", new byte[] { 0, 1, 2, 3, 4 }); ByteBuffer binaryBuffer = ByteBuffer.wrap(new byte[] { 5, 6, 7, 8, 9 }); row.addBinary("binary-bytebuffer", binaryBuffer); row.setNull("null"); row.addDecimal("decimal", BigDecimal.valueOf(12345, 3)); return row; }
assertEquals(5, table.getSchema().getColumnCount());
/** * Test creating and deleting a table through a KuduClient. */ @Test(timeout = 100000) public void testCreateDeleteTable() throws Exception { // Check that we can create a table. client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); assertFalse(client.getTablesList().getTablesList().isEmpty()); assertTrue(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can delete it. client.deleteTable(TABLE_NAME); assertFalse(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can re-recreate it, with a different schema. List<ColumnSchema> columns = new ArrayList<>(basicSchema.getColumns()); columns.add(new ColumnSchema.ColumnSchemaBuilder("one more", Type.STRING).build()); Schema newSchema = new Schema(columns); client.createTable(TABLE_NAME, newSchema, getBasicCreateTableOptions()); // Check that we can open a table and see that it has the new schema. KuduTable table = client.openTable(TABLE_NAME); assertEquals(newSchema.getColumnCount(), table.getSchema().getColumnCount()); assertTrue(table.getPartitionSchema().isSimpleRangePartitioning()); // Check that the block size parameter we specified in the schema is respected. assertEquals(4096, newSchema.getColumn("column3_s").getDesiredBlockSize()); assertEquals(ColumnSchema.Encoding.DICT_ENCODING, newSchema.getColumn("column3_s").getEncoding()); assertEquals(ColumnSchema.CompressionAlgorithm.LZ4, newSchema.getColumn("column3_s").getCompressionAlgorithm()); }
@Test public void testSetMin() { PartialRow partialRow = getPartialRowWithAllTypes(); for (int i = 0; i < partialRow.getSchema().getColumnCount(); i++) { partialRow.setMin(i); } assertEquals(false, partialRow.getBoolean("bool")); assertEquals(Byte.MIN_VALUE, partialRow.getByte("int8")); assertEquals(Short.MIN_VALUE, partialRow.getShort("int16")); assertEquals(Integer.MIN_VALUE, partialRow.getInt("int32")); assertEquals(Long.MIN_VALUE, partialRow.getLong("int64")); assertEquals(Long.MIN_VALUE, partialRow.getLong("timestamp")); assertEquals(-Float.MAX_VALUE, partialRow.getFloat("float"), 0.0f); assertEquals(-Double.MAX_VALUE, partialRow.getDouble("double"), 0.0); assertEquals("", partialRow.getString("string")); assertArrayEquals(new byte[0], partialRow.getBinaryCopy("binary-array")); assertArrayEquals(new byte[0], partialRow.getBinaryCopy("binary-bytebuffer")); assertEquals(BigDecimal.valueOf(-99999, 3), partialRow.getDecimal("decimal")); }