public void addDeletion(IndexType indexType, String id) { add(client.prepareDelete(indexType, id).request()); }
private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<Integer> ruleIds = items .stream() .map(i -> Integer.parseInt(i.getDocId())) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds, r -> { bulkIndexer.add(newRuleDocIndexRequest(r)); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); ruleIds.remove(r.getId()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. ruleIds.forEach(ruleId -> { bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString()); bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); }); return bulkIndexer.stop(); }
private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), i -> { remaining.remove(i.getId()); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
private void doIndex(Iterator<IssueDoc> issues, Size size, IndexingListener listener) { BulkIndexer bulk = createBulkIndexer(size, listener); bulk.start(); while (issues.hasNext()) { IssueDoc issue = issues.next(); bulk.add(newIndexRequest(issue)); } bulk.stop(); }
private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<RuleExtensionId> docIds = items .stream() .map(RuleIndexer::explodeRuleExtensionDocId) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. r -> { RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getRuleId()); docIds.remove(docId); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. docIds.forEach(docId -> bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, docId.getId(), String.valueOf(docId.getRuleId()))); return bulkIndexer.stop(); }
public void addDeletion(IndexType indexType, String id, @Nullable String routing) { add(client.prepareDelete(indexType, id).setRouting(routing).request()); }
@Test public void log_requests_when_TRACE_level_is_enabled() { logTester.setLevel(LoggerLevel.TRACE); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, new FakeListener()); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.addDeletion(INDEX_TYPE_FAKE, "foo"); indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); assertThat(logTester.logs(LoggerLevel.TRACE) .stream() .filter(log -> log.contains("Bulk[2 index requests on fakes/fake, 1 delete requests on fakes/fake]")) .count()).isNotZero(); }
private void doIndex(BulkIndexer bulk, ViewDoc viewDoc, boolean needClearCache) { bulk.add(newIndexRequest(viewDoc)); if (needClearCache) { clearLookupCache(viewDoc.uuid()); } }
private IndexingResult doIndexIssueItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByIssueKey) { if (itemsByIssueKey.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, itemsByIssueKey.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(itemsByIssueKey.keySet())) { while (issues.hasNext()) { IssueDoc issue = issues.next(); bulkIndexer.add(newIndexRequest(issue)); itemsByIssueKey.removeAll(issue.getId()); } } // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
@Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulk.start(); // index all definitions and system extensions if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) { dbClient.ruleDao().scrollIndexingRules(dbSession, dto -> { bulk.add(newRuleDocIndexRequest(dto)); bulk.add(newRuleExtensionDocIndexRequest(dto)); }); } // index all organization extensions if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) { dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> bulk.add(newRuleExtensionDocIndexRequest(dto))); } bulk.stop(); } }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener); bulkIndexer.start(); Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size())); Set<String> remaining = new HashSet<>(branchUuids); for (String branchUuid : branchUuids) { // TODO allow scrolling multiple projects at the same time dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> { ComponentDto dto = context.getResultObject(); bulkIndexer.add(newIndexRequest(toDocument(dto))); remaining.remove(dto.projectUuid()); }); } // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. remaining.forEach(projectUuid -> addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid)); return bulkIndexer.stop(); }
@Test public void index_documents() { BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR); indexer.start(); indexer.add(newIndexRequest(42)); indexer.add(newIndexRequest(78)); // request is not sent yet assertThat(count()).isEqualTo(0); // send remaining requests indexer.stop(); assertThat(count()).isEqualTo(2); }
private void doIndex(Size size, @Nullable String projectUuid) { try (DbSession dbSession = dbClient.openSession(false); ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); while (rowIt.hasNext()) { ProjectMeasures doc = rowIt.next(); bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(doc))); } bulkIndexer.stop(); } }
@Test public void listener_is_not_called_with_errors() { FakeListener listener = new FakeListener(); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.add(new IndexRequest("index_does_not_exist", "index_does_not_exist", "bar").source(emptyMap())); indexer.stop(); assertThat(listener.calledDocIds).containsExactly(new DocId(INDEX_TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isEqualTo(1); assertThat(listener.calledResult.getTotal()).isEqualTo(2); }
@Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { ListMultimap<String, String> organizationUuidsByUserUuid = ArrayListMultimap.create(); dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByUserUuid::put); BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.userDao().scrollAll(dbSession, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. u -> bulkIndexer.add(newIndexRequest(u, organizationUuidsByUserUuid))); bulkIndexer.stop(); } }
@Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar))); bulkIndexer.stop(); } }
/** * @param projectUuid the uuid of the project to analyze, or {@code null} if all content should be indexed.<br/> * <b>Warning:</b> only use {@code null} during startup. */ private void doIndexByProjectUuid(@Nullable String projectUuid, Size bulkSize) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, bulkSize); bulk.start(); try (DbSession dbSession = dbClient.openSession(false)) { dbClient.componentDao() .scrollForIndexing(dbSession, projectUuid, context -> { ComponentDto dto = context.getResultObject(); bulk.add(newIndexRequest(toDocument(dto))); }); } bulk.stop(); }
@Test public void large_indexing() { // index has one replica assertThat(replicas()).isEqualTo(1); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.LARGE); indexer.start(); // replicas are temporarily disabled assertThat(replicas()).isEqualTo(0); for (int i = 0; i < 10; i++) { indexer.add(newIndexRequest(i)); } IndexingResult result = indexer.stop(); assertThat(result.isSuccess()).isTrue(); assertThat(result.getSuccess()).isEqualTo(10); assertThat(result.getFailures()).isEqualTo(0); assertThat(result.getTotal()).isEqualTo(10); assertThat(count()).isEqualTo(10); // replicas are re-enabled assertThat(replicas()).isEqualTo(1); }
@Test public void listener_is_called_even_if_deleting_a_doc_that_does_not_exist() { FakeListener listener = new FakeListener(); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); assertThat(listener.calledDocIds) .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo"), new DocId(INDEX_TYPE_FAKE, "bar")); assertThat(listener.calledResult.getSuccess()).isEqualTo(2); assertThat(listener.calledResult.getTotal()).isEqualTo(2); }
private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) { IndexingResult result = new IndexingResult(); for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) { String ruleProfileUUid = entry.getKey(); EsQueueDto item = entry.getValue(); IndexingResult profileResult; RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE) .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); } else { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i))); profileResult = bulkIndexer.stop(); } if (profileResult.isSuccess()) { deleteQueueDto(dbSession, item); } result.add(profileResult); } return result; }