@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener); bulkIndexer.start(); Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size())); Set<String> remaining = new HashSet<>(branchUuids); for (String branchUuid : branchUuids) { // TODO allow scrolling multiple projects at the same time dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> { ComponentDto dto = context.getResultObject(); bulkIndexer.add(newIndexRequest(toDocument(dto))); remaining.remove(dto.projectUuid()); }); } // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. remaining.forEach(projectUuid -> addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid)); return bulkIndexer.stop(); }
private IndexingListener newListener(Collection<EsQueueDto> items) { return new OneToManyResilientIndexingListener(db.getDbClient(), db.getSession(), items); }
private IndexingResult doIndexProjectItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByProjectUuid) { if (itemsByProjectUuid.isEmpty()) { return new IndexingResult(); } // one project, referenced by es_queue.doc_id = many issues IndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, itemsByProjectUuid.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); for (String projectUuid : itemsByProjectUuid.keySet()) { // TODO support loading of multiple projects in a single SQL request try (IssueIterator issues = issueIteratorFactory.createForProject(projectUuid)) { if (issues.hasNext()) { do { IssueDoc doc = issues.next(); bulkIndexer.add(newIndexRequest(doc)); } while (issues.hasNext()); } else { // project does not exist or has no issues. In both case // all the documents related to this project are deleted. addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid); } } } return bulkIndexer.stop(); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { // The items are to be deleted if (items.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.REGULAR, listener); bulkIndexer.start(); items.forEach(i -> { String projectUuid = i.getDocId(); addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid); }); return bulkIndexer.stop(); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener); bulkIndexer.start(); Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size())); Set<String> remaining = new HashSet<>(branchUuids); for (String branchUuid : branchUuids) { // TODO allow scrolling multiple projects at the same time dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> { ComponentDto dto = context.getResultObject(); bulkIndexer.add(newIndexRequest(toDocument(dto))); remaining.remove(dto.projectUuid()); }); } // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. remaining.forEach(projectUuid -> addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid)); return bulkIndexer.stop(); }
private IndexingResult doIndexProjectItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByProjectUuid) { if (itemsByProjectUuid.isEmpty()) { return new IndexingResult(); } // one project, referenced by es_queue.doc_id = many issues IndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, itemsByProjectUuid.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); for (String projectUuid : itemsByProjectUuid.keySet()) { // TODO support loading of multiple projects in a single SQL request try (IssueIterator issues = issueIteratorFactory.createForProject(projectUuid)) { if (issues.hasNext()) { do { IssueDoc doc = issues.next(); bulkIndexer.add(newIndexRequest(doc)); } while (issues.hasNext()); } else { // project does not exist or has no issues. In both case // all the documents related to this project are deleted. addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid); } } } return bulkIndexer.stop(); }