/** * Delete all the documents matching the given search request. This method is blocking. * Index is refreshed, so docs are not searchable as soon as method is executed. * <p> * Note that the parameter indexType could be removed if progress logs are not needed. */ public static IndexingResult delete(EsClient client, IndexType indexType, SearchRequestBuilder searchRequest) { BulkIndexer bulk = new BulkIndexer(client, indexType, Size.REGULAR); bulk.start(); bulk.addDeletion(searchRequest); return bulk.stop(); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); List<BulkIndexer> bulkIndexers = items.stream() .map(EsQueueDto::getDocType) .distinct() .map(IndexType::parse) .filter(indexTypes::contains) .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) .collect(Collectors.toList()); if (bulkIndexers.isEmpty()) { return result; } bulkIndexers.forEach(BulkIndexer::start); PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet()); permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { remainingProjectUuids.remove(p.getProjectUuid()); bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType()))); }); // the remaining references on projects that don't exist in db. They must // be deleted from index. remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid))); bulkIndexers.forEach(b -> result.add(b.stop())); return result; }
/** * Index a single document. * <p/> * The views lookup cache will be cleared */ public void index(ViewDoc viewDoc) { BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, Size.REGULAR); bulk.start(); doIndex(bulk, viewDoc, true); bulk.stop(); }
public void deleteByKeys(String projectUuid, Collection<String> issueKeys) { if (issueKeys.isEmpty()) { return; } BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, issueKey, projectUuid)); bulkIndexer.stop(); }
private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) { IndexingResult result = new IndexingResult(); for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) { String ruleProfileUUid = entry.getKey(); EsQueueDto item = entry.getValue(); IndexingResult profileResult; RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE) .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); } else { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i))); profileResult = bulkIndexer.stop(); } if (profileResult.isSuccess()) { deleteQueueDto(dbSession, item); } result.add(profileResult); } return result; }
private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) { return new BulkIndexer(esClient, INDEX_TYPE_VIEW, bulkSize, listener); } }
public void addDeletion(IndexType indexType, String id) { add(client.prepareDelete(indexType, id).request()); }
private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX_TYPE_COMPONENT) .setQuery(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) .setRouting(projectUuid); bulkIndexer.addDeletion(searchRequest); }
/** * Completely remove a index with all types */ public void clearIndex(IndexType indexType) { BulkIndexer.delete(esClient, indexType, esClient.prepareSearch(indexType.getIndex()).setQuery(matchAllQuery())); }
@VisibleForTesting void index(ComponentDto... docs) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR); bulk.start(); Arrays.stream(docs) .map(ComponentIndexer::toDocument) .map(ComponentIndexer::newIndexRequest) .forEach(bulk::add); bulk.stop(); }
/** * This is based on the fact that a WebService is only calling {@link ViewIndexer#delete(DbSession, Collection)} * So the resiliency is only taking in account a deletion of view component * A safety check is done by not deleting any component that still exist in database. * * This should not occur but prevent any misuse on this resiliency */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { return new IndexingResult(); } Set<String> views = items .stream() .map(EsQueueDto::getDocId) .collect(toHashSet(items.size())); BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); // Safety check to remove all views that may not have been deleted views.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, views)); views.forEach(v -> bulkIndexer.addDeletion(INDEX_TYPE_VIEW, v)); return bulkIndexer.stop(); }
private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) { IndexingResult result = new IndexingResult(); for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) { String ruleProfileUUid = entry.getKey(); EsQueueDto item = entry.getValue(); IndexingResult profileResult; RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE) .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); } else { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i))); profileResult = bulkIndexer.stop(); } if (profileResult.isSuccess()) { deleteQueueDto(dbSession, item); } result.add(profileResult); } return result; }
private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) { return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE, size, listener); }
public void addDeletion(IndexType indexType, String id, @Nullable String routing) { add(client.prepareDelete(indexType, id).setRouting(routing).request()); }
private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ISSUE) .setRouting(projectUuid) .setQuery(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid))); bulkIndexer.addDeletion(search); }
@Override protected void after() { if (isCustom) { // delete non-core indices String[] existingIndices = SHARED_NODE.client().admin().indices().prepareGetIndex().get().getIndices(); Stream.of(existingIndices) .filter(i -> !CORE_INDICES_NAMES.contains(i)) .forEach(EsTester::deleteIndexIfExists); } BulkIndexer.delete(client(), new IndexType("_all", ""), client().prepareSearch("_all").setQuery(matchAllQuery())); }
public void delete(String projectUuid, Collection<String> disabledComponentUuids) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR); bulk.start(); disabledComponentUuids.forEach(uuid -> bulk.addDeletion(INDEX_TYPE_COMPONENT, uuid, projectUuid)); bulk.stop(); }
@Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar))); bulkIndexer.stop(); } }
private void index(Collection<IndexPermissions> authorizations, Stream<AuthorizationScope> scopes, Size bulkSize) { if (authorizations.isEmpty()) { return; } // index each authorization in each scope scopes.forEach(scope -> { IndexType indexType = scope.getIndexType(); BulkIndexer bulkIndexer = new BulkIndexer(esClient, indexType, bulkSize); bulkIndexer.start(); authorizations.stream() .filter(scope.getProjectPredicate()) .map(dto -> newIndexRequest(dto, indexType)) .forEach(bulkIndexer::add); bulkIndexer.stop(); }); }