private void extractFromSource(SearchHit hit) { Map<String, Object> map = hit.getSourceAsMap(); for (Map.Entry<String, Object> entry : map.entrySet()) { String jsonPath = entry.getKey(); Object entryValue = entry.getValue(); setFieldIfExists(jsonPath, entryValue); } } }
protected void mergeSourceAndAddAliases(Map<String,Object> secondTableHitSource, SearchHit searchHit,String t1Alias,String t2Alias) { Map<String,Object> results = mapWithAliases(searchHit.getSourceAsMap(), t1Alias); results.putAll(mapWithAliases(secondTableHitSource, t2Alias)); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(results); }
private Object getFieldValue(SearchHit hit, String fieldName) { Map<String,Object> sourceAsMap = hit.getSourceAsMap(); if(fieldName.contains(".")){ String[] split = fieldName.split("\\."); return Util.searchPathInMap(sourceAsMap, split); } else if(sourceAsMap.containsKey(fieldName)){ return sourceAsMap.get(fieldName); } return null; }
public static Object getFieldValue(SearchHit hit,String field){ return deepSearchInMap(hit.getSourceAsMap(),field); }
public ComperableHitResult(SearchHit hit , String[] fieldsOrder ,String seperator) { this.hit = hit; Map<String, Object> hitAsMap = hit.getSourceAsMap(); this.flattenMap = new HashMap<>(); List<String> results = new ArrayList<>(); this.isAllNull = true; for(int i = 0 ; i< fieldsOrder.length ;i++){ String field = fieldsOrder[i]; Object result = Util.deepSearchInMap(hitAsMap,field); if(result == null){ results.add(""); } else { this.isAllNull = false; results.add(result.toString()); this.flattenMap.put(field,result); } } this.comperator = Joiner.on(seperator).join(results); }
public SearchResult(SearchResponse resp) { SearchHits hits = resp.getHits(); this.total = hits.getTotalHits(); results = new ArrayList<>(hits.getHits().length); for (SearchHit searchHit : hits.getHits()) { if (searchHit.getSourceAsMap() != null) { results.add(searchHit.getSourceAsMap()); } else if (searchHit.getFields() != null) { Map<String, DocumentField> fields = searchHit.getFields(); results.add(toFieldsMap(fields)); } } }
protected SearchHit createUnmachedResult( List<Field> secondTableReturnedFields, int docId, String t1Alias, String t2Alias, SearchHit hit) { String unmatchedId = hit.getId() + "|0"; Text unamatchedType = new Text(hit.getType() + "|null"); SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, hit.getFields()); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hit.getSourceAsMap()); Map<String,Object> emptySecondTableHitSource = createNullsSource(secondTableReturnedFields); mergeSourceAndAddAliases(emptySecondTableHitSource, searchHit,t1Alias,t2Alias); return searchHit; }
private void fillInternalSearchHits(List<SearchHit> unionHits, SearchHit[] hits, Map<String, String> fieldNameToAlias) { for(SearchHit hit : hits){ SearchHit searchHit = new SearchHit(currentId, hit.getId(), new Text(hit.getType()), hit.getFields()); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map<String, Object> sourceAsMap = hit.getSourceAsMap(); if(!fieldNameToAlias.isEmpty()){ updateFieldNamesToAlias(sourceAsMap, fieldNameToAlias); } searchHit.getSourceAsMap().putAll(sourceAsMap); currentId++; unionHits.add(searchHit); } }
private void fillMinusHitsFromOneField(String fieldName, Set<Object> fieldValues, SearchHit someHit) { List<SearchHit> minusHitsList = new ArrayList<>(); int currentId = 1; for(Object result : fieldValues){ Map<String,DocumentField> fields = new HashMap<>(); ArrayList<Object> values = new ArrayList<Object>(); values.add(result); fields.put(fieldName,new DocumentField(fieldName, values)); SearchHit searchHit = new SearchHit(currentId,currentId+"", new Text(someHit.getType()), fields); searchHit.sourceRef(someHit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map<String, Object> sourceAsMap = new HashMap<>(); sourceAsMap.put(fieldName,result); searchHit.getSourceAsMap().putAll(sourceAsMap); currentId++; minusHitsList.add(searchHit); } int totalSize = currentId - 1; SearchHit[] unionHitsArr = minusHitsList.toArray(new SearchHit[totalSize]); this.minusHits = new SearchHits(unionHitsArr, totalSize,1.0f); }
private String getComparisonKey(List<Map.Entry<Field, Field>> t1ToT2FieldsComparison, SearchHit hit, boolean firstTable, Map<String, List<Object>> optimizationTermsFilterStructure) { String key = ""; Map<String, Object> sourceAsMap = hit.getSourceAsMap(); for (Map.Entry<Field, Field> t1ToT2 : t1ToT2FieldsComparison) { //todo: change to our function find if key contains '.' String name; if (firstTable) name = t1ToT2.getKey().getName(); else name = t1ToT2.getValue().getName(); Object data = deepSearchInMap(sourceAsMap, name); if (firstTable && useQueryTermsFilterOptimization) { updateOptimizationData(optimizationTermsFilterStructure, data, t1ToT2.getValue().getName()); } if (data == null) key += "|null|"; else key += "|" + data.toString() + "|"; } return key; }
private SearchHit getMergedHit(int currentCombinedResults, String t1Alias, String t2Alias, SearchHit hitFromFirstTable, SearchHit matchedHit) { onlyReturnedFields(matchedHit.getSourceAsMap(), nestedLoopsRequest.getSecondTable().getReturnedFields(),nestedLoopsRequest.getSecondTable().getOriginalSelect().isSelectAll()); SearchHit searchHit = new SearchHit(currentCombinedResults, hitFromFirstTable.getId() + "|" + matchedHit.getId(), new Text(hitFromFirstTable.getType() + "|" + matchedHit.getType()), hitFromFirstTable.getFields()); searchHit.sourceRef(hitFromFirstTable.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hitFromFirstTable.getSourceAsMap()); mergeSourceAndAddAliases(matchedHit.getSourceAsMap(), searchHit, t1Alias, t2Alias); return searchHit; }
private void fillMinusHitsFromResults(Set<ComperableHitResult> comperableHitResults) { int currentId = 1; List<SearchHit> minusHitsList = new ArrayList<>(); for(ComperableHitResult result : comperableHitResults){ ArrayList<Object> values = new ArrayList<Object>(); values.add(result); SearchHit originalHit = result.getOriginalHit(); SearchHit searchHit = new SearchHit(currentId,originalHit.getId(), new Text(originalHit.getType()), originalHit.getFields()); searchHit.sourceRef(originalHit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map<String, Object> sourceAsMap = result.getFlattenMap(); for(Map.Entry<String,String> entry : this.builder.getFirstTableFieldToAlias().entrySet()){ if(sourceAsMap.containsKey(entry.getKey())){ Object value = sourceAsMap.get(entry.getKey()); sourceAsMap.remove(entry.getKey()); sourceAsMap.put(entry.getValue(),value); } } searchHit.getSourceAsMap().putAll(sourceAsMap); currentId++; minusHitsList.add(searchHit); } int totalSize = currentId - 1; SearchHit[] unionHitsArr = minusHitsList.toArray(new SearchHit[totalSize]); this.minusHits = new SearchHits(unionHitsArr, totalSize,1.0f); }
/** * Get a list of a specific field from all indexed documents. */ public <T> List<T> getDocumentFieldValues(IndexType indexType, final String fieldNameToReturn) { return getDocuments(indexType) .stream() .map(input -> (T) input.getSourceAsMap().get(fieldNameToReturn)) .collect(Collectors.toList()); }
private List<String> createHeadersAndFillDocsMap(boolean flat, SearchHit[] hits, List<Map<String, Object>> docsAsMap) { Set<String> csvHeaders = new HashSet<>(); for(SearchHit hit : hits){ Map<String, Object> doc = hit.getSourceAsMap(); Map<String, DocumentField> fields = hit.getFields(); for(DocumentField searchHitField : fields.values()){
private List<String> createHeadersAndFillDocsMap(boolean flat, SearchHit[] hits, List<Map<String, Object>> docsAsMap) { Set<String> headers = new LinkedHashSet<>(); if (this.queryAction instanceof DefaultQueryAction) { headers.addAll(((DefaultQueryAction) this.queryAction).getFieldNames()); } for (SearchHit hit : hits) { Map<String, Object> doc = hit.getSourceAsMap(); Map<String, DocumentField> fields = hit.getFields(); for (DocumentField searchHitField : fields.values()) { doc.put(searchHitField.getName(), searchHitField.getValue()); } if (this.includeScore) { doc.put("_score", hit.getScore()); } if (this.includeType) { doc.put("_type", hit.getType()); } if (this.includeId) { doc.put("_id", hit.getId()); } mergeHeaders(headers, doc, flat); docsAsMap.add(doc); } return new ArrayList<>(headers); }
public static String hitsAsStringResult(SearchHits results, MetaSearchResult metaResults) throws IOException { if(results == null) return null; Object[] searchHits; searchHits = new Object[(int) results.getTotalHits()]; int i = 0; for(SearchHit hit : results) { HashMap<String,Object> value = new HashMap<>(); value.put("_id",hit.getId()); value.put("_type", hit.getType()); value.put("_score", hit.getScore()); value.put("_source", hit.getSourceAsMap()); searchHits[i] = value; i++; } HashMap<String,Object> hits = new HashMap<>(); hits.put("total",results.getTotalHits()); hits.put("max_score",results.getMaxScore()); hits.put("hits",searchHits); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); builder.startObject(); builder.field("took", metaResults.getTookImMilli()); builder.field("timed_out",metaResults.isTimedOut()); builder.field("_shards", ImmutableMap.of("total", metaResults.getTotalNumOfShards(), "successful", metaResults.getSuccessfulShards() , "failed", metaResults.getFailedShards())); builder.field("hits",hits) ; builder.endObject(); return BytesReference.bytes(builder).utf8ToString(); } }
private void createKeyToResultsAndFillOptimizationStructure(Map<String,Map<String, List<Object>>> optimizationTermsFilterStructure, TableInJoinRequestBuilder firstTableRequest) { List<SearchHit> firstTableHits = fetchAllHits(firstTableRequest); int resultIds = 1; for (SearchHit hit : firstTableHits) { HashMap<String, List<Map.Entry<Field, Field>>> comparisons = this.hashJoinComparisonStructure.getComparisons(); for (Map.Entry<String, List<Map.Entry<Field, Field>>> comparison : comparisons.entrySet()) { String comparisonID = comparison.getKey(); List<Map.Entry<Field, Field>> t1ToT2FieldsComparison = comparison.getValue(); String key = getComparisonKey(t1ToT2FieldsComparison, hit, true, optimizationTermsFilterStructure.get(comparisonID)); //int docid , id SearchHit searchHit = new SearchHit(resultIds, hit.getId(), new Text(hit.getType()), hit.getFields()); searchHit.sourceRef(hit.getSourceRef()); onlyReturnedFields(searchHit.getSourceAsMap(), firstTableRequest.getReturnedFields(),firstTableRequest.getOriginalSelect().isSelectAll()); resultIds++; this.hashJoinComparisonStructure.insertIntoComparisonHash(comparisonID, key, searchHit); } } }
/** * Returns the active users (at most 3) who are associated to the given SCM account. This method can be used * to detect user conflicts. */ public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount, String organizationUuid) { List<UserDoc> result = new ArrayList<>(); if (!StringUtils.isEmpty(scmAccount)) { SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) .setQuery(boolQuery().must(matchAllQuery()).filter( boolQuery() .must(termQuery(FIELD_ACTIVE, true)) .must(termQuery(FIELD_ORGANIZATION_UUIDS, organizationUuid)) .should(termQuery(FIELD_LOGIN, scmAccount)) .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)))) .setSize(3); for (SearchHit hit : request.get().getHits().getHits()) { result.add(new UserDoc(hit.getSourceAsMap())); } } return result; }
private MultiSearchRequest createMultiSearchRequest(int multiSearchMaxSize, Where connectedWhere, SearchHit[] hits, Select secondTableSelect, Where originalWhere, int currentIndex) throws SqlParseException { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); for(int i = currentIndex ; i < currentIndex + multiSearchMaxSize && i< hits.length ; i++ ){ Map<String, Object> hitFromFirstTableAsMap = hits[i].getSourceAsMap(); Where newWhere = Where.newInstance(); if(originalWhere!=null) newWhere.addWhere(originalWhere);
private int combineResultsFromMultiResponses(List<SearchHit> combinedResults, int totalLimit, int currentCombinedResults, SearchHit[] hits, int currentIndex, MultiSearchRequest multiSearchRequest) { MultiSearchResponse.Item[] responses = client.multiSearch(multiSearchRequest).actionGet().getResponses(); String t1Alias = nestedLoopsRequest.getFirstTable().getAlias(); String t2Alias = nestedLoopsRequest.getSecondTable().getAlias(); for(int j =0 ; j < responses.length && currentCombinedResults < totalLimit ; j++){ SearchHit hitFromFirstTable = hits[currentIndex+j]; onlyReturnedFields(hitFromFirstTable.getSourceAsMap(), nestedLoopsRequest.getFirstTable().getReturnedFields(),nestedLoopsRequest.getFirstTable().getOriginalSelect().isSelectAll()); SearchResponse multiItemResponse = responses[j].getResponse(); updateMetaSearchResults(multiItemResponse); //todo: if responseForHit.getHits.length < responseForHit.getTotalHits(). need to fetch more! SearchHits responseForHit = multiItemResponse.getHits(); if(responseForHit.getHits().length == 0 && nestedLoopsRequest.getJoinType() == SQLJoinTableSource.JoinType.LEFT_OUTER_JOIN){ SearchHit unmachedResult = createUnmachedResult(nestedLoopsRequest.getSecondTable().getReturnedFields(), currentCombinedResults, t1Alias, t2Alias, hitFromFirstTable); combinedResults.add(unmachedResult); currentCombinedResults++; continue; } for(SearchHit matchedHit : responseForHit.getHits() ){ SearchHit searchHit = getMergedHit(currentCombinedResults, t1Alias, t2Alias, hitFromFirstTable, matchedHit); combinedResults.add(searchHit); currentCombinedResults++; if(currentCombinedResults >= totalLimit) break; } if(currentCombinedResults >= totalLimit) break; } return currentCombinedResults; }