private void apply(Operation op) { try { OperationResponse response = session.apply(op); if (response != null && response.hasRowError()) { LOG.info("Write operation failed: {}", response.getRowError()); } } catch (KuduException ex) { LOG.warn("Write operation failed", ex); } } }
private boolean hasRowErrorAndReport(OperationResponse resp) { if (resp != null && resp.hasRowError()) { reportError("The following RPC " + resp.getOperation().getRow() + " returned this error: " + resp.getRowError(), null); return true; } if (resp == null) { return false; } sharedWriteTimestamp = resp.getWriteTimestampRaw(); return false; } }
@Test(timeout = 100000) public void singleTabletTest() throws Exception { String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis(); harness.getClient().createTable(tableName, basicSchema, getBasicCreateTableOptions()); table = harness.getClient().openTable(tableName); AsyncKuduSession session = harness.getAsyncClient().newSession(); // Insert 3 rows to play with. for (int i = 0; i < 3; i++) { session.apply(createInsert(i)).join(DEFAULT_SLEEP); } // Try a single dupe row insert with AUTO_FLUSH_SYNC. Insert dupeForZero = createInsert(0); OperationResponse resp = session.apply(dupeForZero).join(DEFAULT_SLEEP); assertTrue(resp.hasRowError()); assertTrue(resp.getRowError().getOperation() == dupeForZero); // Now try inserting two dupes and one good row, make sure we get only two errors back. dupeForZero = createInsert(0); Insert dupeForTwo = createInsert(2); session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH); session.apply(dupeForZero); session.apply(dupeForTwo); session.apply(createInsert(4)); List<OperationResponse> responses = session.flush().join(DEFAULT_SLEEP); List<RowError> errors = OperationResponse.collectErrors(responses); assertEquals(2, errors.size()); assertTrue(errors.get(0).getOperation() == dupeForZero); assertTrue(errors.get(1).getOperation() == dupeForTwo); }
@Override void updateStatistics(Statistics statistics, BatchResponse response) { String tabletId = this.getTablet().getTabletId(); String tableName = this.getTable().getName(); TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId); if (response == null) { tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, operations.size()); tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1); return; } tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1); for (OperationResponse opResponse : response.getIndividualResponses()) { if (opResponse.hasRowError()) { tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1); } else { tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1); } } tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationsSizeBytes()); }
private Boolean processResponse(List<OperationResponse> operationResponses) { Boolean isOk = operationResponses.isEmpty(); for(OperationResponse operationResponse : operationResponses) { logResponseError(operationResponse.getRowError()); } return isOk; }
@Override public Deferred<OperationResponse> call(OperationResponse resp) throws Exception { client.updateLastPropagatedTimestamp(resp.getWriteTimestampRaw()); return Deferred.fromResult(resp); } };
@Override public Object call(Exception e) throws Exception { if (e instanceof KuduException) { Status status = ((KuduException) e).getStatus(); RowError rowError = new RowError(status, operation); return new OperationResponse(0, null, 0, operation, rowError); } return e; } }
@Test(timeout = 10000) public void testUpsert() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); // Test an Upsert that acts as an Insert. assertFalse(session.apply(createUpsert(table, 1, 1, false)).hasRowError()); List<String> rowStrings = scanTableToStrings(table); assertEquals(1, rowStrings.size()); assertEquals( "INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=true", rowStrings.get(0)); // Test an Upsert that acts as an Update. assertFalse(session.apply(createUpsert(table, 1, 2, false)).hasRowError()); rowStrings = scanTableToStrings(table); assertEquals( "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " + "STRING column3_s=a string, BOOL column4_b=true", rowStrings.get(0)); }
assertTrue(client.hasLastPropagatedTimestamp()); assertEquals(client.getLastPropagatedTimestamp(), response.getWriteTimestampRaw()); clockValues = HTTimestampToPhysicalAndLogical(client.getLastPropagatedTimestamp()); LOG.debug("Clock value after write[" + i + "]: " + new Date(clockValues[0] / 1000).toString() assertTrue(client.hasLastPropagatedTimestamp()); assertEquals(client.getLastPropagatedTimestamp(), response.getWriteTimestampRaw()); clockValues = HTTimestampToPhysicalAndLogical(client.getLastPropagatedTimestamp()); LOG.debug("Clock value after write[" + i + "]: " + new Date(clockValues[0] / 1000).toString()
new OperationResponse(currentOperation.deadlineTracker.getElapsedMillis(), tsUUID, writeTimestamp, currentOperation, rowError));
private void flush() throws IOException { try { // context.getStats().startWait(); List<OperationResponse> responses = session.flush(); for (OperationResponse response : responses) { if (response.hasRowError()) { throw new IOException(response.getRowError().toString()); } } } catch (Exception e) { throw new IOException(e); } finally { // context.getStats().stopWait(); } }
@Override void updateStatistics(Statistics statistics, OperationResponse response) { String tabletId = this.getTablet().getTabletId(); String tableName = this.getTable().getName(); TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId); if (response == null) { tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1); tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1); return; } tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1); if (response.hasRowError()) { // If ignoreAllDuplicateRows is set, the already_present exception will be // discarded and wont't be recorded here tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1); } else { tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1); } tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationSizeBytes()); }
error = new RowError(Status.RuntimeError(failure.getMessage()), operation); OperationResponse response = new OperationResponse(0, null, 0, operation, error);
if (response != null && response.hasRowError()) { flowFileFailures.put(flowFile, response.getRowError()); break;
@Test(timeout = 100000) public void testIgnoreAllDuplicateRows() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); session.setIgnoreAllDuplicateRows(true); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } for (SessionConfiguration.FlushMode mode : SessionConfiguration.FlushMode.values()) { session.setFlushMode(mode); for (int i = 0; i < 10; i++) { OperationResponse resp = session.apply(createInsert(table, i)); if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC) { assertFalse(resp.hasRowError()); } } if (mode == SessionConfiguration.FlushMode.MANUAL_FLUSH) { List<OperationResponse> responses = session.flush(); for (OperationResponse resp : responses) { assertFalse(resp.hasRowError()); } } else if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { while (session.hasPendingOperations()) { Thread.sleep(100); } assertEquals(0, session.countPendingErrors()); } } }
@Override Pair<OperationResponse, Object> deserialize(CallResponse callResponse, String tsUUID) throws KuduException { Tserver.WriteResponsePB.Builder builder = Tserver.WriteResponsePB.newBuilder(); readProtobuf(callResponse.getPBMessage(), builder); Tserver.WriteResponsePB.PerRowErrorPB error = null; if (builder.getPerRowErrorsCount() != 0) { error = builder.getPerRowErrors(0); if (ignoreAllDuplicateRows && error.getError().getCode() == WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT) { error = null; } } OperationResponse response = new OperationResponse(deadlineTracker.getElapsedMillis(), tsUUID, builder.getTimestamp(), this, error); return new Pair<OperationResponse, Object>( response, builder.hasError() ? builder.getError() : null); }
/** * Utility method that collects all the row errors from the given list of responses. * @param responses a list of operation responses to collect the row errors from * @return a combined list of row errors */ public static List<RowError> collectErrors(List<OperationResponse> responses) { List<RowError> errors = new ArrayList<>(responses.size()); for (OperationResponse resp : responses) { if (resp.hasRowError()) { errors.add(resp.getRowError()); } } return errors; }
@Test(timeout = 100000) public void testCreateTableWithConcurrentInsert() throws Exception { KuduTable table = client.createTable( TABLE_NAME, createManyStringsSchema(), getBasicCreateTableOptions().setWait(false)); // Insert a row. // // It's very likely that the tablets are still being created, but the client // should transparently retry the insert (and associated master lookup) // until the operation succeeds. Insert insert = table.newInsert(); insert.getRow().addString("key", "key_0"); insert.getRow().addString("c1", "c1_0"); insert.getRow().addString("c2", "c2_0"); KuduSession session = client.newSession(); OperationResponse resp = session.apply(insert); assertFalse(resp.hasRowError()); // This won't do anything useful (i.e. if the insert succeeds, we know the // table has been created), but it's here for additional code coverage. assertTrue(client.isCreateTableDone(TABLE_NAME)); }
@Test(timeout = 10000) public void testInsertAutoFlushSyncNonCoveredRange() throws Exception { CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange(); createOptions.setNumReplicas(1); client.createTable(tableName, basicSchema, createOptions); KuduTable table = client.openTable(tableName); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50); for (int key : nonCoveredKeys) { OperationResponse response = session.apply(createBasicSchemaInsert(table, key)); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isNotFound()); } }
@Test(timeout = 100000) public void testBasicOps() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build())); OperationResponse resp = session.apply(createInsert(table, 0)); assertTrue(resp.hasRowError()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); for (int i = 10; i < 20; i++) { session.apply(createInsert(table, i)); } session.flush(); assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build())); }