@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); List<BulkIndexer> bulkIndexers = items.stream() .map(EsQueueDto::getDocType) .distinct() .map(IndexType::parse) .filter(indexTypes::contains) .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) .collect(Collectors.toList()); if (bulkIndexers.isEmpty()) { return result; } bulkIndexers.forEach(BulkIndexer::start); PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet()); permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { remainingProjectUuids.remove(p.getProjectUuid()); bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType()))); }); // the remaining references on projects that don't exist in db. They must // be deleted from index. remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid))); bulkIndexers.forEach(b -> result.add(b.stop())); return result; }
/** * This is based on the fact that a WebService is only calling {@link ViewIndexer#delete(DbSession, Collection)} * So the resiliency is only taking in account a deletion of view component * A safety check is done by not deleting any component that still exist in database. * * This should not occur but prevent any misuse on this resiliency */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } Set<String> views = items .stream() .map(EsQueueDto::getDocId) .collect(toHashSet(items.size())); BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); // Safety check to remove all views that may not have been deleted views.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, views)); views.forEach(v -> bulkIndexer.addDeletion(INDEX_TYPE_VIEW, v)); return bulkIndexer.stop(); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); List<String> projectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toArrayList(items.size())); Iterator<String> it = projectUuids.iterator(); while (it.hasNext()) { String projectUuid = it.next(); try (ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { while (rowIt.hasNext()) { bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(rowIt.next()))); it.remove(); } } } // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(INDEX_TYPE_PROJECT_MEASURES, projectUuid, projectUuid)); return bulkIndexer.stop(); }
private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), i -> { remaining.remove(i.getId()); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<Integer> ruleIds = items .stream() .map(i -> Integer.parseInt(i.getDocId())) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds, r -> { bulkIndexer.add(newRuleDocIndexRequest(r)); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); ruleIds.remove(r.getId()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. ruleIds.forEach(ruleId -> { bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString()); bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); }); return bulkIndexer.stop(); }
/** * @return the number of items that have been successfully indexed */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } Set<String> uuids = items .stream() .map(EsQueueDto::getDocId) .collect(toHashSet(items.size())); ListMultimap<String, String> organizationUuidsByUserUuid = ArrayListMultimap.create(); dbClient.organizationMemberDao().selectForUserIndexing(dbSession, uuids, organizationUuidsByUserUuid::put); BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); dbClient.userDao().scrollByUuids(dbSession, uuids, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. u -> { uuids.remove(u.getUuid()); bulkIndexer.add(newIndexRequest(u, organizationUuidsByUserUuid)); }); // the remaining uuids reference rows that don't exist in db. They must // be deleted from index. uuids.forEach(uuid -> bulkIndexer.addDeletion(INDEX_TYPE_USER, uuid)); return bulkIndexer.stop(); }
private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<RuleExtensionId> docIds = items .stream() .map(RuleIndexer::explodeRuleExtensionDocId) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. r -> { RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getRuleId()); docIds.remove(docId); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. docIds.forEach(docId -> bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, docId.getId(), String.valueOf(docId.getRuleId()))); return bulkIndexer.stop(); }
private IndexingListener newListener(Collection<EsQueueDto> items) { return new OneToOneResilientIndexingListener(db.getDbClient(), db.getSession(), items); }
private IndexingResult doIndexIssueItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByIssueKey) { if (itemsByIssueKey.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, itemsByIssueKey.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(itemsByIssueKey.keySet())) { while (issues.hasNext()) { IssueDoc issue = issues.next(); bulkIndexer.add(newIndexRequest(issue)); itemsByIssueKey.removeAll(issue.getId()); } } // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); List<BulkIndexer> bulkIndexers = items.stream() .map(EsQueueDto::getDocType) .distinct() .map(IndexType::parse) .filter(indexTypes::contains) .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) .collect(Collectors.toList()); if (bulkIndexers.isEmpty()) { return result; } bulkIndexers.forEach(BulkIndexer::start); PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet()); permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { remainingProjectUuids.remove(p.getProjectUuid()); bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType()))); }); // the remaining references on projects that don't exist in db. They must // be deleted from index. remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid))); bulkIndexers.forEach(b -> result.add(b.stop())); return result; }
/** * This is based on the fact that a WebService is only calling {@link ViewIndexer#delete(DbSession, Collection)} * So the resiliency is only taking in account a deletion of view component * A safety check is done by not deleting any component that still exist in database. * * This should not occur but prevent any misuse on this resiliency */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } Set<String> views = items .stream() .map(EsQueueDto::getDocId) .collect(toHashSet(items.size())); BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); // Safety check to remove all views that may not have been deleted views.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, views)); views.forEach(v -> bulkIndexer.addDeletion(INDEX_TYPE_VIEW, v)); return bulkIndexer.stop(); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); List<String> projectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toArrayList(items.size())); Iterator<String> it = projectUuids.iterator(); while (it.hasNext()) { String projectUuid = it.next(); try (ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { while (rowIt.hasNext()) { bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(rowIt.next()))); it.remove(); } } } // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(INDEX_TYPE_PROJECT_MEASURES, projectUuid, projectUuid)); return bulkIndexer.stop(); }
private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), i -> { remaining.remove(i.getId()); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<Integer> ruleIds = items .stream() .map(i -> Integer.parseInt(i.getDocId())) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds, r -> { bulkIndexer.add(newRuleDocIndexRequest(r)); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); ruleIds.remove(r.getId()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. ruleIds.forEach(ruleId -> { bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString()); bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); }); return bulkIndexer.stop(); }
/** * @return the number of items that have been successfully indexed */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } Set<String> uuids = items .stream() .map(EsQueueDto::getDocId) .collect(toHashSet(items.size())); ListMultimap<String, String> organizationUuidsByUserUuid = ArrayListMultimap.create(); dbClient.organizationMemberDao().selectForUserIndexing(dbSession, uuids, organizationUuidsByUserUuid::put); BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); dbClient.userDao().scrollByUuids(dbSession, uuids, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. u -> { uuids.remove(u.getUuid()); bulkIndexer.add(newIndexRequest(u, organizationUuidsByUserUuid)); }); // the remaining uuids reference rows that don't exist in db. They must // be deleted from index. uuids.forEach(uuid -> bulkIndexer.addDeletion(INDEX_TYPE_USER, uuid)); return bulkIndexer.stop(); }
private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<RuleExtensionId> docIds = items .stream() .map(RuleIndexer::explodeRuleExtensionDocId) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. r -> { RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getRuleId()); docIds.remove(docId); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. docIds.forEach(docId -> bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, docId.getId(), docId.getId())); return bulkIndexer.stop(); }
private IndexingResult doIndexIssueItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByIssueKey) { if (itemsByIssueKey.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, itemsByIssueKey.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(itemsByIssueKey.keySet())) { while (issues.hasNext()) { IssueDoc issue = issues.next(); bulkIndexer.add(newIndexRequest(issue)); itemsByIssueKey.removeAll(issue.getId()); } } // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }