@Override public void deleteRows(Block rowIds) { Schema schema = table.getSchema(); KuduSession session = clientSession.newSession(); session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND); try { try { for (int i = 0; i < rowIds.getPositionCount(); i++) { int len = rowIds.getSliceLength(i); Slice slice = rowIds.getSlice(i, 0, len); PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes()); Delete delete = table.newDelete(); RowHelper.copyPrimaryKey(schema, row, delete.getRow()); session.apply(delete); } } finally { session.close(); } } catch (KuduException e) { throw new RuntimeException(e); } }
private void flushKuduSession(final KuduSession kuduSession, boolean close, final List<RowError> rowErrors) throws KuduException { final List<OperationResponse> responses = close ? kuduSession.close() : kuduSession.flush(); if (kuduSession.getFlushMode() == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { rowErrors.addAll(Arrays.asList(kuduSession.getPendingErrors().getRowErrors())); } else { responses.stream() .filter(OperationResponse::hasRowError) .map(OperationResponse::getRowError) .forEach(rowErrors::add); } }
protected KuduSession getKuduSession(final KuduClient client) { final KuduSession kuduSession = client.newSession(); kuduSession.setMutationBufferSpace(batchSize); kuduSession.setFlushMode(flushMode); if (operationType == OperationType.INSERT_IGNORE) { kuduSession.setIgnoreAllDuplicateRows(true); } return kuduSession; }
@Test(timeout = 100000) public void testBasicOps() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build())); OperationResponse resp = session.apply(createInsert(table, 0)); assertTrue(resp.hasRowError()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); for (int i = 10; i < 20; i++) { session.apply(createInsert(table, i)); } session.flush(); assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build())); }
@Override public void createSchema(KuduClient client, String schemaName) { if (DEFAULT_SCHEMA.equals(schemaName)) { throw new SchemaAlreadyExistsException(schemaName); } else { try { KuduTable schemasTable = getSchemasTable(client); KuduSession session = client.newSession(); try { Upsert upsert = schemasTable.newUpsert(); upsert.getRow().addString(0, schemaName); session.apply(upsert); } finally { session.close(); } } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } } }
/** * Test collecting errors from multiple tablets. * @throws Exception */ @Test(timeout = 100000) public void multiTabletTest() throws Exception { String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis(); createFourTabletsTableWithNineRows(harness.getAsyncClient(), tableName, DEFAULT_SLEEP); table = harness.getClient().openTable(tableName); KuduSession session = harness.getClient().newSession(); session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); int dupRows = 3; session.apply(createInsert(12)); session.apply(createInsert(22)); session.apply(createInsert(32)); session.flush(); RowErrorsAndOverflowStatus reos = session.getPendingErrors(); assertEquals(dupRows, reos.getRowErrors().length); assertEquals(0, session.countPendingErrors()); }
session.setFlushMode(flushMode); KuduTable table = client.openTable(TABLE_NAME); for (int i = 0; i < 3; i++) { row.addString("c2", "c2_" + j); row.addString("c3", "c3_" + j); session.apply(insert); session.flush(); session.close();
@Override public void endRecord() throws IOException { try { session.apply(insert); recordsSinceFlush++; if (recordsSinceFlush == FLUSH_FREQUENCY) { flush(); recordsSinceFlush = 0; } insert = null; } catch (Exception e) { throw new IOException(e); } }
@Override public void run() { session.setExternalConsistencyMode(ExternalConsistencyMode.CLIENT_PROPAGATED); while (KEEP_RUNNING_LATCH.getCount() > 0) { try { OperationResponse resp = session.apply(createBasicSchemaInsert(table, currentRowKey)); if (hasRowErrorAndReport(resp)) { return; List<OperationResponse> responses = session.flush(); if (responses != null) { for (OperationResponse batchedResp : responses) { session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); } else { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
/** * Load a table of default schema with the specified number of records, in ascending key order. */ public static void loadDefaultTable(KuduClient client, String tableName, int numRows) throws KuduException { KuduTable table = client.openTable(tableName); KuduSession session = client.newSession(); for (int i = 0; i < numRows; i++) { Insert insert = createBasicSchemaInsert(table, i); session.apply(insert); } session.flush(); session.close(); }
session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); for (int i = 0; i < rows; i++) { Insert insert = table.newInsert(); row.addFloat(3, i + 0.01f); row.addString(4, ("Row " + i)); session.apply(insert);
@Test(timeout = 100000) public void testIgnoreAllDuplicateRows() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); session.setIgnoreAllDuplicateRows(true); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } for (SessionConfiguration.FlushMode mode : SessionConfiguration.FlushMode.values()) { session.setFlushMode(mode); for (int i = 0; i < 10; i++) { OperationResponse resp = session.apply(createInsert(table, i)); if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC) { assertFalse(resp.hasRowError()); } } if (mode == SessionConfiguration.FlushMode.MANUAL_FLUSH) { List<OperationResponse> responses = session.flush(); for (OperationResponse resp : responses) { assertFalse(resp.hasRowError()); } } else if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { while (session.hasPendingOperations()) { Thread.sleep(100); } assertEquals(0, session.countPendingErrors()); } } }
private void closeSession() { try { session.close(); } catch (KuduException e) { throw new RuntimeException(e); } } }
row.addInt("addNullable", 101); row.setNull("addNullableDef"); session.apply(insert); session.flush(); RowError[] rowErrors = session.getPendingErrors().getRowErrors(); assertEquals(String.format("row errors: %s", Arrays.toString(rowErrors)), 0, rowErrors.length);
/** * Regression test for KUDU-1226. Calls to session.flush() concurrent with AUTO_FLUSH_BACKGROUND * can end up giving ConvertBatchToListOfResponsesCB a list with nulls if a tablet was already * flushed. Only happens with multiple tablets. */ @Test(timeout = 10000) public void testConcurrentFlushes() throws Exception { CreateTableOptions builder = getBasicCreateTableOptions(); int numTablets = 4; int numRowsPerTablet = 100; // Create a 4 tablets table split on 1000, 2000, and 3000. for (int i = 1; i < numTablets; i++) { PartialRow split = basicSchema.newPartialRow(); split.addInt(0, i * numRowsPerTablet); builder.addSplitRow(split); } KuduTable table = client.createTable(tableName, basicSchema, builder); // Configure the session to background flush as often as it can (every 1ms). KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setFlushInterval(1); // Fill each tablet in parallel 1 by 1 then flush. Without the fix this would quickly get an // NPE. for (int i = 0; i < numRowsPerTablet; i++) { for (int j = 0; j < numTablets; j++) { session.apply(createInsert(table, i + (numRowsPerTablet * j))); } session.flush(); } }
@Test public void testBasicWorkflow() throws Exception { KuduTable table = client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); dropConnectionsAndExpireToken(); KuduSession session = client.newSession(); session.setTimeoutMillis(OP_TIMEOUT_MS); session.apply(createBasicSchemaInsert(table, 1)); session.flush(); RowErrorsAndOverflowStatus errors = session.getPendingErrors(); assertFalse(errors.isOverflowed()); assertEquals(0, session.countPendingErrors()); dropConnectionsAndExpireToken(); KuduTable scanTable = client.openTable(TABLE_NAME); AsyncKuduScanner scanner = new AsyncKuduScanner.AsyncKuduScannerBuilder(asyncClient, scanTable) .scanRequestTimeout(OP_TIMEOUT_MS) .build(); assertEquals(1, countRowsInScan(scanner)); dropConnectionsAndExpireToken(); client.deleteTable(TABLE_NAME); assertFalse(client.tableExists(TABLE_NAME)); } }
public KuduRecordWriterImpl(OperatorContext context, KuduClient client, String name) { this.client = client; this.name = name; this.context = context; session = client.newSession(); session.setFlushMode(FlushMode.MANUAL_FLUSH); }
@Override public void applyRandomMutations(List<Row> planned) throws Exception { KuduConnection connection = getConnection(); KuduSession session = connection.getSession(); KuduTable table = connection.getTable(config.getString(TABLE_CONFIG_NAME)); List<Operation> operations = extractOperations(planned, table); for (Operation operation : operations) { session.apply(operation); } // Wait until all operations have completed before checking for errors. while (session.hasPendingOperations()) { Thread.sleep(1); } // Fail fast on any error applying mutations if (session.countPendingErrors() > 0) { RowError firstError = session.getPendingErrors().getRowErrors()[0]; String errorMessage = String.format("Kudu output error '%s' during operation '%s' at tablet server '%s'", firstError.getErrorStatus(), firstError.getOperation(), firstError.getTsUUID()); throw new RuntimeException(errorMessage); } }
@Override public void init() throws DBException { this.tableName = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); initClient(); this.session = client.newSession(); if (getProperties().getProperty(SYNC_OPS_OPT) != null && getProperties().getProperty(SYNC_OPS_OPT).equals("false")) { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); this.session.setMutationBufferSpace( getIntFromProp(getProperties(), BUFFER_NUM_OPS_OPT, BUFFER_NUM_OPS_DEFAULT)); } else { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_SYNC); } try { this.kuduTable = client.openTable(tableName); this.schema = kuduTable.getSchema(); } catch (Exception e) { throw new DBException("Could not open a table because of:", e); } }