private void apply(Operation op) { try { OperationResponse response = session.apply(op); if (response != null && response.hasRowError()) { LOG.info("Write operation failed: {}", response.getRowError()); } } catch (KuduException ex) { LOG.warn("Write operation failed", ex); } } }
private void flush() throws IOException { try { // context.getStats().startWait(); List<OperationResponse> responses = session.flush(); for (OperationResponse response : responses) { if (response.hasRowError()) { throw new IOException(response.getRowError().toString()); } } } catch (Exception e) { throw new IOException(e); } finally { // context.getStats().stopWait(); } }
if (response != null && response.hasRowError()) {
/** * Utility method that collects all the row errors from the given list of responses. * @param responses a list of operation responses to collect the row errors from * @return a combined list of row errors */ public static List<RowError> collectErrors(List<OperationResponse> responses) { List<RowError> errors = new ArrayList<>(responses.size()); for (OperationResponse resp : responses) { if (resp.hasRowError()) { errors.add(resp.getRowError()); } } return errors; }
private boolean hasRowErrorAndReport(OperationResponse resp) { if (resp != null && resp.hasRowError()) { reportError("The following RPC " + resp.getOperation().getRow() + " returned this error: " + resp.getRowError(), null); return true; } if (resp == null) { return false; } sharedWriteTimestamp = resp.getWriteTimestampRaw(); return false; } }
@Override void updateStatistics(Statistics statistics, BatchResponse response) { String tabletId = this.getTablet().getTabletId(); String tableName = this.getTable().getName(); TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId); if (response == null) { tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, operations.size()); tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1); return; } tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1); for (OperationResponse opResponse : response.getIndividualResponses()) { if (opResponse.hasRowError()) { tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1); } else { tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1); } } tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationsSizeBytes()); }
@Test(timeout = 10000) public void testUpsert() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); // Test an Upsert that acts as an Insert. assertFalse(session.apply(createUpsert(table, 1, 1, false)).hasRowError()); List<String> rowStrings = scanTableToStrings(table); assertEquals(1, rowStrings.size()); assertEquals( "INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=true", rowStrings.get(0)); // Test an Upsert that acts as an Update. assertFalse(session.apply(createUpsert(table, 1, 2, false)).hasRowError()); rowStrings = scanTableToStrings(table); assertEquals( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=true", rowStrings.get(0)); }
@Override void updateStatistics(Statistics statistics, OperationResponse response) { String tabletId = this.getTablet().getTabletId(); String tableName = this.getTable().getName(); TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId); if (response == null) { tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1); tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1); return; } tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1); if (response.hasRowError()) { // If ignoreAllDuplicateRows is set, the already_present exception will be // discarded and wont't be recorded here tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1); } else { tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1); } tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationSizeBytes()); }
@Test(timeout = 100000) public void testIgnoreAllDuplicateRows() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); session.setIgnoreAllDuplicateRows(true); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } for (SessionConfiguration.FlushMode mode : SessionConfiguration.FlushMode.values()) { session.setFlushMode(mode); for (int i = 0; i < 10; i++) { OperationResponse resp = session.apply(createInsert(table, i)); if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC) { assertFalse(resp.hasRowError()); } } if (mode == SessionConfiguration.FlushMode.MANUAL_FLUSH) { List<OperationResponse> responses = session.flush(); for (OperationResponse resp : responses) { assertFalse(resp.hasRowError()); } } else if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { while (session.hasPendingOperations()) { Thread.sleep(100); } assertEquals(0, session.countPendingErrors()); } } }
@Test(timeout = 10000) public void testInsertManualFlushResponseOrder() throws Exception { CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange(); createOptions.setNumReplicas(1); client.createTable(tableName, basicSchema, createOptions); KuduTable table = client.openTable(tableName); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // Insert a batch of some valid and some invalid. for (int i = 0; i < 10; i++) { assertNull(session.apply(createBasicSchemaInsert(table, 100 + i * 10))); assertNull(session.apply(createBasicSchemaInsert(table, 200 + i * 10))); } List<OperationResponse> results = session.flush(); assertEquals(20, results.size()); for (int i = 0; i < 20; i++) { OperationResponse result = results.get(i); if (i % 2 == 0) { assertTrue(result.hasRowError()); assertTrue(result.getRowError().getErrorStatus().isNotFound()); } else { assertTrue(!result.hasRowError()); } } }
@Test(timeout = 10000) public void testInsertAutoFlushSyncNonCoveredRange() throws Exception { CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange(); createOptions.setNumReplicas(1); client.createTable(tableName, basicSchema, createOptions); KuduTable table = client.openTable(tableName); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50); for (int key : nonCoveredKeys) { OperationResponse response = session.apply(createBasicSchemaInsert(table, key)); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isNotFound()); } }
@Test(timeout = 100000) public void testBackgroundErrors() throws Exception { try { AsyncKuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setFlushInterval(10); Batch.injectTabletServerErrorAndLatency(makeTabletServerError(), 0); OperationResponse resp = session.apply(createInsert(1)).join(DEFAULT_SLEEP); assertTrue(resp.hasRowError()); assertTrue( resp.getRowError().getErrorStatus() .getMessage().contains(getTabletServerErrorMessage())); assertEquals(1, session.countPendingErrors()); } finally { Batch.injectTabletServerErrorAndLatency(null, 0); } }
/** Regression test for a failure to correctly handle a timeout when flushing a batch. */ @Test public void testInsertIntoUnavailableTablet() throws Exception { harness.killAllTabletServers(); try { AsyncKuduSession session = client.newSession(); session.setTimeoutMillis(1); OperationResponse response = session.apply(createInsert(1)).join(); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isTimedOut()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); Insert insert = createInsert(1); session.apply(insert); List<OperationResponse> responses = session.flush().join(); assertEquals(1, responses.size()); assertTrue(responses.get(0).getRowError().getErrorStatus().isTimedOut()); } finally { harness.startAllTabletServers(); } }
@Test(timeout = 100000) public void singleTabletTest() throws Exception { String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis(); harness.getClient().createTable(tableName, basicSchema, getBasicCreateTableOptions()); table = harness.getClient().openTable(tableName); AsyncKuduSession session = harness.getAsyncClient().newSession(); // Insert 3 rows to play with. for (int i = 0; i < 3; i++) { session.apply(createInsert(i)).join(DEFAULT_SLEEP); } // Try a single dupe row insert with AUTO_FLUSH_SYNC. Insert dupeForZero = createInsert(0); OperationResponse resp = session.apply(dupeForZero).join(DEFAULT_SLEEP); assertTrue(resp.hasRowError()); assertTrue(resp.getRowError().getOperation() == dupeForZero); // Now try inserting two dupes and one good row, make sure we get only two errors back. dupeForZero = createInsert(0); Insert dupeForTwo = createInsert(2); session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH); session.apply(dupeForZero); session.apply(dupeForTwo); session.apply(createInsert(4)); List<OperationResponse> responses = session.flush().join(DEFAULT_SLEEP); List<RowError> errors = OperationResponse.collectErrors(responses); assertEquals(2, errors.size()); assertTrue(errors.get(0).getOperation() == dupeForZero); assertTrue(errors.get(1).getOperation() == dupeForTwo); }
@Test(timeout = 100000) public void testCreateTableWithConcurrentInsert() throws Exception { KuduTable table = client.createTable( TABLE_NAME, createManyStringsSchema(), getBasicCreateTableOptions().setWait(false)); // Insert a row. // // It's very likely that the tablets are still being created, but the client // should transparently retry the insert (and associated master lookup) // until the operation succeeds. Insert insert = table.newInsert(); insert.getRow().addString("key", "key_0"); insert.getRow().addString("c1", "c1_0"); insert.getRow().addString("c2", "c2_0"); KuduSession session = client.newSession(); OperationResponse resp = session.apply(insert); assertFalse(resp.hasRowError()); // This won't do anything useful (i.e. if the insert succeeds, we know the // table has been created), but it's here for additional code coverage. assertTrue(client.isCreateTableDone(TABLE_NAME)); }
@Test(timeout = 10000) public void testInsertAutoFlushBackgrounNonCoveredRange() throws Exception { CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange(); createOptions.setNumReplicas(1); client.createTable(tableName, basicSchema, createOptions); KuduTable table = client.openTable(tableName); AsyncKuduSession session = asyncClient.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50); for (int key : nonCoveredKeys) { OperationResponse result = session.apply(createBasicSchemaInsert(table, key)).join(5000); assertTrue(result.hasRowError()); assertTrue(result.getRowError().getErrorStatus().isNotFound()); } RowErrorsAndOverflowStatus errors = session.getPendingErrors(); assertEquals(nonCoveredKeys.size(), errors.getRowErrors().length); for (RowError error : errors.getRowErrors()) { assertTrue(error.getErrorStatus().isNotFound()); } // Insert a batch of some valid and some invalid. for (int key = 90; key < 110; key++) { session.apply(createBasicSchemaInsert(table, key)); } session.flush(); errors = session.getPendingErrors(); assertEquals(10, errors.getRowErrors().length); for (RowError error : errors.getRowErrors()) { assertTrue(error.getErrorStatus().isNotFound()); } }
@Test(timeout = 100000) public void testBasicOps() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build())); OperationResponse resp = session.apply(createInsert(table, 0)); assertTrue(resp.hasRowError()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); for (int i = 10; i < 20; i++) { session.apply(createInsert(table, i)); } session.flush(); assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build())); }
@Test public void testConnectionRefused() throws Exception { CreateTableOptions options = getBasicCreateTableOptions(); KuduTable table = client.createTable( "testConnectionRefused-" + System.currentTimeMillis(), basicSchema, options); // Warm up the caches. assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build())); // Make it impossible to use Kudu. harness.killAllTabletServers(); // Create a scan with a short timeout. KuduScanner scanner = client.newScannerBuilder(table).scanRequestTimeout(1000).build(); // Check it fails. try { while (scanner.hasMoreRows()) { scanner.nextRows(); fail("The scan should timeout"); } } catch (NonRecoverableException ex) { assertTrue(ex.getStatus().isTimedOut()); } // Try the same thing with an insert. KuduSession session = client.newSession(); session.setTimeoutMillis(1000); OperationResponse response = session.apply(createBasicSchemaInsert(table, 1)); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isTimedOut()); }
/** * This test writes 3 rows, kills the leader, then tries to write another 3 rows. Finally it * counts to make sure we have 6 of them. * * This test won't run if we didn't start the cluster. */ @Test(timeout = 100000) public void testFailover() throws Exception { KuduSession session = harness.getClient().newSession(); for (int i = 0; i < 3; i++) { session.apply(createBasicSchemaInsert(table, i)); } // Make sure the rows are in there before messing things up. AsyncKuduScanner scanner = harness.getAsyncClient().newScannerBuilder(table).build(); assertEquals(3, countRowsInScan(scanner)); harness.killTabletLeader(table); for (int i = 3; i < 6; i++) { OperationResponse resp = session.apply(createBasicSchemaInsert(table, i)); if (resp.hasRowError()) { fail("Encountered a row error " + resp.getRowError()); } } scanner = harness.getAsyncClient().newScannerBuilder(table).build(); assertEquals(6, countRowsInScan(scanner)); } }
assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isTimedOut());