@Override public void onFinish(IndexingResult result) { if (result.getFailures() > 0) { throw new IllegalStateException(String.format("Unrecoverable indexation failures: %d errors among %d requests", result.getFailures(), result.getTotal())); } } };
@Test public void test_empty() { assertThat(underTest.getFailures()).isEqualTo(0); assertThat(underTest.getSuccess()).isEqualTo(0); assertThat(underTest.getTotal()).isEqualTo(0); assertThat(underTest.getSuccessRatio()).isEqualTo(1.0, DOUBLE_OFFSET); assertThat(underTest.isSuccess()).isTrue(); }
@Test public void correctness_even_with_no_data() { assertThat(underTest.getFailures()).isEqualTo(0); assertThat(underTest.getSuccess()).isEqualTo(0); assertThat(underTest.getTotal()).isEqualTo(0); assertThat(underTest.getSuccessRatio()).isEqualTo(1.0); assertThat(underTest.isSuccess()).isTrue(); } }
@Test public void test_failure() { underTest.incrementRequests(); underTest.incrementRequests(); assertThat(underTest.getFailures()).isEqualTo(2); assertThat(underTest.getSuccess()).isEqualTo(0); assertThat(underTest.getTotal()).isEqualTo(2); assertThat(underTest.getSuccessRatio()).isEqualTo(0.0, DOUBLE_OFFSET); assertThat(underTest.isSuccess()).isFalse(); }
@Test public void test_success() { underTest.incrementRequests(); underTest.incrementRequests(); underTest.incrementSuccess(); underTest.incrementSuccess(); assertThat(underTest.getFailures()).isEqualTo(0); assertThat(underTest.getSuccess()).isEqualTo(2); assertThat(underTest.getTotal()).isEqualTo(2); assertThat(underTest.getSuccessRatio()).isEqualTo(1.0, DOUBLE_OFFSET); assertThat(underTest.isSuccess()).isTrue(); }
@Test public void errors_during_indexing_are_recovered() { ComponentDto project = db.components().insertPrivateProject(); ComponentDto file = db.components().insertComponent(newFileDto(project)); es.lockWrites(INDEX_TYPE_COMPONENT); IndexingResult result = indexProject(project, PROJECT_CREATION); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(2L); // index is still read-only, fail to recover result = recover(); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(2L); assertThat(es.countDocuments(INDEX_TYPE_COMPONENT)).isEqualTo(0); es.unlockWrites(INDEX_TYPE_COMPONENT); result = recover(); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(0L); assertThatIndexContainsOnly(project, file); }
@Test public void test_partial_failure() { underTest.incrementRequests(); underTest.incrementRequests(); underTest.incrementRequests(); underTest.incrementRequests(); underTest.incrementSuccess(); assertThat(underTest.getFailures()).isEqualTo(3); assertThat(underTest.getSuccess()).isEqualTo(1); assertThat(underTest.getTotal()).isEqualTo(4); assertThat(underTest.getSuccessRatio()).isEqualTo(0.25, DOUBLE_OFFSET); assertThat(underTest.isSuccess()).isFalse(); }
@Test public void errors_during_indexing_are_recovered() { ComponentDto project = createAndIndexPublicProject(); es.lockWrites(INDEX_TYPE_FOO_AUTH); IndexingResult result = indexPermissions(project, PERMISSION_CHANGE); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); // index is still read-only, fail to recover result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); assertThatAuthIndexHasSize(0); assertThatEsQueueTableHasSize(1); es.unlockWrites(INDEX_TYPE_FOO_AUTH); result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(0L); verifyAnyoneAuthorized(project); assertThatEsQueueTableHasSize(0); }
@Test public void errors_during_project_deletion_are_recovered() { addIssueToIndex("P1", "I1"); assertThatIndexHasSize(1); es.lockWrites(INDEX_TYPE_ISSUE); IndexingResult result = indexProject("P1", ProjectIndexer.Cause.PROJECT_DELETION); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); // index is still read-only, fail to recover result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); assertThatIndexHasSize(1); es.unlockWrites(INDEX_TYPE_ISSUE); result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(0L); assertThatIndexHasSize(0); }
@Test public void large_indexing() { // index has one replica assertThat(replicas()).isEqualTo(1); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.LARGE); indexer.start(); // replicas are temporarily disabled assertThat(replicas()).isEqualTo(0); for (int i = 0; i < 10; i++) { indexer.add(newIndexRequest(i)); } IndexingResult result = indexer.stop(); assertThat(result.isSuccess()).isTrue(); assertThat(result.getSuccess()).isEqualTo(10); assertThat(result.getFailures()).isEqualTo(0); assertThat(result.getTotal()).isEqualTo(10); assertThat(count()).isEqualTo(10); // replicas are re-enabled assertThat(replicas()).isEqualTo(1); }
@Test public void errors_during_indexing_are_recovered() { ComponentDto project = db.components().insertPrivateProject(); es.lockWrites(INDEX_TYPE_PROJECT_MEASURES); IndexingResult result = indexProject(project, PROJECT_CREATION); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); // index is still read-only, fail to recover result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0); assertThatEsQueueTableHasSize(1); es.unlockWrites(INDEX_TYPE_PROJECT_MEASURES); result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(0L); assertThatEsQueueTableHasSize(0); assertThatIndexContainsOnly(project); }
@VisibleForTesting void recover() { try (DbSession dbSession = dbClient.openSession(false)) { Profiler profiler = Profiler.create(LOGGER).start(); long beforeDate = system2.now() - minAgeInMs; IndexingResult result = new IndexingResult(); Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); while (!items.isEmpty()) { IndexingResult loopResult = new IndexingResult(); groupItemsByType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); result.add(loopResult); if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) { LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal()); break; } if (loopResult.getTotal() == 0L) { break; } items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); } if (result.getTotal() > 0L) { profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures())); } } catch (Throwable t) { LOGGER.error(LOG_PREFIX + "fail to recover documents", t); } }
@Test public void indexing_recovers_multiple_errors_on_the_same_project() { RuleDefinitionDto rule = db.rules().insert(); ComponentDto project = db.components().insertPrivateProject(organization); ComponentDto file = db.components().insertComponent(newFileDto(project)); IssueDto issue1 = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file)); IssueDto issue2 = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file)); es.lockWrites(INDEX_TYPE_ISSUE); IndexingResult result = indexProject(project.uuid(), ProjectIndexer.Cause.PROJECT_DELETION); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(2L); // index is still read-only, fail to recover result = recover(); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(2L); assertThatIndexHasSize(0); es.unlockWrites(INDEX_TYPE_ISSUE); result = recover(); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(0L); assertThatIndexHasSize(2); assertThatEsQueueTableHasSize(0); }
@Override public void onFinish(IndexingResult result) { if (result.getFailures() > 0) { throw new IllegalStateException(String.format("Unrecoverable indexation failures: %d errors among %d requests", result.getFailures(), result.getTotal())); } } };
@VisibleForTesting void recover() { try (DbSession dbSession = dbClient.openSession(false)) { Profiler profiler = Profiler.create(LOGGER).start(); long beforeDate = system2.now() - minAgeInMs; IndexingResult result = new IndexingResult(); Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); while (!items.isEmpty()) { IndexingResult loopResult = new IndexingResult(); groupItemsByType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); result.add(loopResult); if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) { LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal()); break; } if (loopResult.getTotal() == 0L) { break; } items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); } if (result.getTotal() > 0L) { profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures())); } } catch (Throwable t) { LOGGER.error(LOG_PREFIX + "fail to recover documents", t); } }