/** * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}. * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} * is different from the given {@code metaData}. * This method will throw an exception if the creation or the update fails. * The created {@link IndexService} will not be registered and will be closed immediately. */ public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException { final List<Closeable> closeables = new ArrayList<>(); try { IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); closeables.add(indicesFieldDataCache); IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings); closeables.add(indicesQueryCache); // this will also fail if some plugin fails etc. which is nice since we can verify that early final IndexService service = createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList()); closeables.add(() -> service.close("metadata verification", false)); service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true); if (metaData.equals(metaDataUpdate) == false) { service.updateMetaData(metaData, metaDataUpdate); } } finally { IOUtils.close(closeables); } }
dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE, false);
indexService.mapperService().merge(indexMetaData, MergeReason.MAPPING_RECOVERY, true);
@Override public ClusterTasksResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState, List<PutMappingClusterStateUpdateRequest> tasks) throws Exception { Map<Index, MapperService> indexMapperServices = new HashMap<>(); ClusterTasksResult.Builder<PutMappingClusterStateUpdateRequest> builder = ClusterTasksResult.builder(); try { for (PutMappingClusterStateUpdateRequest request : tasks) { try { for (Index index : request.indices()) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMapperServices.containsKey(indexMetaData.getIndex()) == false) { MapperService mapperService = indicesService.createIndexMapperService(indexMetaData); indexMapperServices.put(index, mapperService); // add mappings for all types, we need them for cross-type validation mapperService.merge(indexMetaData, MergeReason.MAPPING_RECOVERY, request.updateAllTypes()); } } currentState = applyRequest(currentState, request, indexMapperServices); builder.success(request); } catch (Exception e) { builder.failure(request, e); } } return builder.build(currentState); } finally { IOUtils.close(indexMapperServices.values()); } }
mappingUpdateConsumer.accept(mapping.key, mapping.value); indexShard.mapperService().merge(sourceMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, mapperRegistry, () -> null); mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, false);
existingSource = existingMapper.mappingSource(); DocumentMapper mergedMapper = mapperService.merge(mappingType, mappingUpdateSource, MergeReason.MAPPING_UPDATE, request.updateAllTypes()); CompressedXContent updatedSource = mergedMapper.mappingSource();
throw new ElasticsearchException("Failed to create temporary index for parsing the alias", e); indexService.mapperService().merge(index, MapperService.MergeReason.MAPPING_RECOVERY, false);
mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, request.updateAllTypes()); } catch (Exception e) { removalExtraInfo = "failed on parsing default mapping/mappings on index creation";
/** * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}. * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} is different from the given {@code metaData}. * This method will throw an exception if the creation or the update fails. * The created {@link IndexService} will not be registered and will be closed immediately. */ public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException { final List<Closeable> closeables = new ArrayList<>(); try { IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); closeables.add(indicesFieldDataCache); IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings); closeables.add(indicesQueryCache); // this will also fail if some plugin fails etc. which is nice since we can verify that early final IndexService service = createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList()); closeables.add(() -> service.close("metadata verification", false)); service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true); if (metaData.equals(metaDataUpdate) == false) { service.updateMetaData(metaDataUpdate); } } finally { IOUtils.close(closeables); } }
public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException { // first, add the default mapping if (mappings.containsKey(DEFAULT_MAPPING)) { try { this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes); } catch (Exception e) { throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage()); } } for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) { if (entry.getKey().equals(DEFAULT_MAPPING)) { continue; } try { // apply the default here, its the first time we parse it this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes); } catch (Exception e) { throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage()); } } }
protected void updateMappings(IndexShard shard, IndexMetaData indexMetadata) { shard.indexSettings().updateIndexMetaData(indexMetadata); shard.mapperService().merge(indexMetadata, MapperService.MergeReason.MAPPING_UPDATE, true); }
/** * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}. * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} is different from the given {@code metaData}. * This method will throw an exception if the creation or the update fails. * The created {@link IndexService} will not be registered and will be closed immediately. */ public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException { final List<Closeable> closeables = new ArrayList<>(); try { IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); closeables.add(indicesFieldDataCache); IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings); closeables.add(indicesQueryCache); // this will also fail if some plugin fails etc. which is nice since we can verify that early final IndexService service = createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList()); closeables.add(() -> service.close("metadata verification", false)); service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true); if (metaData.equals(metaDataUpdate) == false) { service.updateMetaData(metaDataUpdate); } } finally { IOUtils.close(closeables); } }
/** * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}. * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} is different from the given {@code metaData}. * This method will throw an exception if the creation or the update fails. * The created {@link IndexService} will not be registered and will be closed immediately. */ public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException { final List<Closeable> closeables = new ArrayList<>(); try { IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); closeables.add(indicesFieldDataCache); IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings); closeables.add(indicesQueryCache); // this will also fail if some plugin fails etc. which is nice since we can verify that early final IndexService service = createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList()); closeables.add(() -> service.close("metadata verification", false)); service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true); if (metaData.equals(metaDataUpdate) == false) { service.updateMetaData(metaData, metaDataUpdate); } } finally { IOUtils.close(closeables); } }
public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) { if (DEFAULT_MAPPING.equals(type)) { // verify we can parse it // NOTE: never apply the default here DocumentMapper mapper = documentParser.parse(type, mappingSource); // still add it as a document mapper so we have it registered and, for example, persisted back into // the cluster meta data if needed, or checked for existence synchronized (this) { mappers = newMapBuilder(mappers).put(type, mapper).map(); } try { defaultMappingSource = mappingSource.string(); } catch (IOException e) { throw new ElasticsearchGenerationException("failed to un-compress", e); } return mapper; } else { synchronized (this) { final boolean applyDefault = // the default was already applied if we are recovering reason != MergeReason.MAPPING_RECOVERY // only apply the default mapping if we don't have the type yet && mappers.containsKey(type) == false; DocumentMapper mergeWith = parse(type, mappingSource, applyDefault); return merge(mergeWith, reason, updateAllTypes); } } }
@Override public ClusterTasksResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState, List<PutMappingClusterStateUpdateRequest> tasks) throws Exception { Map<Index, MapperService> indexMapperServices = new HashMap<>(); ClusterTasksResult.Builder<PutMappingClusterStateUpdateRequest> builder = ClusterTasksResult.builder(); try { for (PutMappingClusterStateUpdateRequest request : tasks) { try { for (Index index : request.indices()) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMapperServices.containsKey(indexMetaData.getIndex()) == false) { MapperService mapperService = indicesService.createIndexMapperService(indexMetaData); indexMapperServices.put(index, mapperService); // add mappings for all types, we need them for cross-type validation mapperService.merge(indexMetaData, MergeReason.MAPPING_RECOVERY, request.updateAllTypes()); } } currentState = applyRequest(currentState, request, indexMapperServices); builder.success(request); } catch (Exception e) { builder.failure(request, e); } } return builder.build(currentState, true); } finally { IOUtils.close(indexMapperServices.values()); } }
@Override public ClusterTasksResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState, List<PutMappingClusterStateUpdateRequest> tasks) throws Exception { Map<Index, MapperService> indexMapperServices = new HashMap<>(); ClusterTasksResult.Builder<PutMappingClusterStateUpdateRequest> builder = ClusterTasksResult.builder(); try { for (PutMappingClusterStateUpdateRequest request : tasks) { try { for (Index index : request.indices()) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMapperServices.containsKey(indexMetaData.getIndex()) == false) { MapperService mapperService = indicesService.createIndexMapperService(indexMetaData); indexMapperServices.put(index, mapperService); // add mappings for all types, we need them for cross-type validation mapperService.merge(indexMetaData, MergeReason.MAPPING_RECOVERY, request.updateAllTypes()); } } currentState = applyRequest(currentState, request, indexMapperServices); builder.success(request); } catch (Exception e) { builder.failure(request, e); } } return builder.build(currentState); } finally { IOUtils.close(indexMapperServices.values()); } }
@Override public ClusterTasksResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState, List<PutMappingClusterStateUpdateRequest> tasks) throws Exception { Map<Index, MapperService> indexMapperServices = new HashMap<>(); ClusterTasksResult.Builder<PutMappingClusterStateUpdateRequest> builder = ClusterTasksResult.builder(); try { for (PutMappingClusterStateUpdateRequest request : tasks) { try { for (Index index : request.indices()) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMapperServices.containsKey(indexMetaData.getIndex()) == false) { MapperService mapperService = indicesService.createIndexMapperService(indexMetaData); indexMapperServices.put(index, mapperService); // add mappings for all types, we need them for cross-type validation mapperService.merge(indexMetaData, MergeReason.MAPPING_RECOVERY, request.updateAllTypes()); } } currentState = applyRequest(currentState, request, indexMapperServices); builder.success(request); } catch (Exception e) { builder.failure(request, e); } } return builder.build(currentState); } finally { IOUtils.close(indexMapperServices.values()); } }
/** * Checks the mappings for compatibility with the current version */ private void checkMappingsCompatibility(IndexMetaData indexMetaData) { Index index = new Index(indexMetaData.getIndex()); Settings settings = indexMetaData.getSettings(); try { SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings); // We cannot instantiate real analysis server at this point because the node might not have // been started yet. However, we don't really need real analyzers at this stage - so we can fake it try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) { try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityLookupService, scriptService, mapperRegistry)) { for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) { MappingMetaData mappingMetaData = cursor.value; mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false); } } } } catch (Exception ex) { // Wrap the inner exception so we have the index name in the exception message throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetaData.getIndex() + "], reason: [" + ex.getMessage() + "]", ex); } }
private boolean processMapping(String index, MapperService mapperService, String mappingType, CompressedXContent mappingSource) throws Throwable { // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the // merge version of it, which it does when refreshing the mappings), and warn log it. boolean requiresRefresh = false; try { DocumentMapper existingMapper = mapperService.documentMapper(mappingType); if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) { String op = existingMapper == null ? "adding" : "updating"; if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) { logger.debug("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string()); } else if (logger.isTraceEnabled()) { logger.trace("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string()); } else { logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index, op, mappingType); } mapperService.merge(mappingType, mappingSource, MapperService.MergeReason.MAPPING_RECOVERY, true); if (!mapperService.documentMapper(mappingType).mappingSource().equals(mappingSource)) { logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index, mappingType, mappingSource, mapperService.documentMapper(mappingType).mappingSource()); requiresRefresh = true; } } } catch (Throwable e) { logger.warn("[{}] failed to add mapping [{}], source [{}]", e, index, mappingType, mappingSource); throw e; } return requiresRefresh; }