Skip to content

Make v2 search case- and accent-insensitive #592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jrhoads merged 1 commit into from
Sep 29, 2026
Merged

Make v2 search case- and accent-insensitive #592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions rorapi/common/queries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import re
import json
import unicodedata
from titlecase import titlecase
from collections import defaultdict

Expand Down Expand Up @@ -187,6 +188,14 @@ def validate(params):
return Errors(errors) if errors else None


def nfc(value):
"""Normalizes a string to Unicode NFC so decomposed (NFD) input matches
the precomposed characters indexed in Elasticsearch"""
if isinstance(value, str):
return unicodedata.normalize("NFC", value)
return value


def build_search_query(params):
"""Builds search query from API parameters"""

Expand All @@ -198,20 +207,21 @@ def build_search_query(params):
del params["all_status"]

if "query.advanced" in params:
qb.add_string_query_advanced(params.get("query.advanced"))
qb.add_string_query_advanced(nfc(params.get("query.advanced")))
elif "query" in params:
ror_id = get_ror_id(params.get("query"))
query = nfc(params.get("query"))
ror_id = get_ror_id(query)
if ror_id is not None:
qb.add_id_query(ror_id)
else:
qb.add_string_query(params.get("query"))
qb.add_string_query(query)
else:
qb.add_match_all_query()

if "filter" in params or (not "all_status" in params):
filters = [
f.split(":")
for f in filter_string_to_list(params.get("filter", ""))
for f in filter_string_to_list(nfc(params.get("filter", "")))
if f
]
# normalize filter values based on casing conventions used in ROR records
Expand Down Expand Up @@ -247,10 +257,10 @@ def build_search_query(params):

qb.add_aggregations(
[
("types", "types"),
("countries", "locations.geonames_details.country_code"),
("continents", "locations.geonames_details.continent_code"),
("statuses", "status"),
("types", "types.raw"),
("countries", "locations.geonames_details.country_code.raw"),
("continents", "locations.geonames_details.continent_code.raw"),
("statuses", "status.raw"),
]
)

Expand Down
78 changes: 78 additions & 0 deletions rorapi/tests/tests_integration/tests_search_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,81 @@ def test_extra_word(self):
}).json()
self.assertTrue(items['number_of_results'] > 0)
self.assertEqual(items['items'][0]['id'], 'https://ror.org/00fbnyb24')


class CaseAndAccentInsensitiveTestCase(SimpleTestCase):
"""Case- and diacritic-insensitive matching (ror-roadmap#175, #398).

Requires an index created from the current index_template_es7.json.
"""

def search(self, **params):
return requests.get(BASE_URL, params).json()

def assert_same_results(self, variants, param='query.advanced'):
results = [self.search(**{param: v}) for v in variants]
baseline = results[0]
self.assertTrue(baseline['number_of_results'] > 0, variants[0])
baseline_ids = {i['id'] for i in baseline['items']}
for variant, result in zip(variants[1:], results[1:]):
self.assertEqual(result['number_of_results'],
baseline['number_of_results'], variant)
self.assertEqual({i['id'] for i in result['items']},
baseline_ids, variant)

def test_keyword_field_case(self):
self.assert_same_results([
'locations.geonames_details.name:Denver',
'locations.geonames_details.name:denver',
'locations.geonames_details.name:DENVER',
])

def test_relationship_type_case(self):
self.assert_same_results([
'relationships.type:child',
'relationships.type:Child',
])

def test_bare_term_case(self):
self.assert_same_results(['Stellenbosch', 'stellenbosch'])

def test_keyword_field_diacritics(self):
self.assert_same_results([
'locations.geonames_details.name:Huế',
'locations.geonames_details.name:Hue',
'locations.geonames_details.name:hue',
])
self.assert_same_results([
'locations.geonames_details.name:Montréal',
'locations.geonames_details.name:Montreal',
])

def test_nfd_and_nfc_input(self):
nfc = 'locations.geonames_details.name:Hu\u1ebf'
nfd = 'locations.geonames_details.name:Hu\u0065\u0302\u0301'
self.assertNotEqual(nfc, nfd)
self.assert_same_results([nfc, nfd])

def test_text_field_diacritics(self):
self.assert_same_results([
'names.value:"Université de Montréal"',
'names.value:"Universite de Montreal"',
'names.value:"universite de montreal"',
])

def test_filter_case(self):
self.assert_same_results(['Denver'], param='query')
upper = requests.get(BASE_URL, {'filter': 'types:Education'}).json()
lower = requests.get(BASE_URL, {'filter': 'types:education'}).json()
self.assertTrue(upper['number_of_results'] > 0)
self.assertEqual(upper['number_of_results'],
lower['number_of_results'])

def test_aggregation_keys_keep_original_casing(self):
# Bucket titles are derived from the raw (un-normalized) keys, e.g. the
# country name is looked up from the upper-case ISO code.
meta = self.search(**{'query.advanced': 'locations.geonames_details.name:denver'})['meta']
countries = {b['id']: b['title'] for b in meta['countries']}
self.assertEqual(countries.get('us'), 'United States')
continents = {b['id']: b['title'] for b in meta['continents']}
self.assertEqual(continents.get('na'), 'North America')
40 changes: 36 additions & 4 deletions rorapi/tests/tests_unit/tests_queries_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,10 @@ class BuildSearchQueryTestCase(SimpleTestCase):

def setUp(self):
self.default_query = \
{'aggs': {'types': {'terms': {'field': 'types', 'size': 10, 'min_doc_count': 1}},
'countries': {'terms': {'field': 'locations.geonames_details.country_code', 'size': 10, 'min_doc_count': 1}},
'continents': {'terms': {'field': 'locations.geonames_details.continent_code', 'size': 10, 'min_doc_count': 1}},
'statuses': {'terms': {'field': 'status', 'size': 10, 'min_doc_count': 1}}},
{'aggs': {'types': {'terms': {'field': 'types.raw', 'size': 10, 'min_doc_count': 1}},
'countries': {'terms': {'field': 'locations.geonames_details.country_code.raw', 'size': 10, 'min_doc_count': 1}},
'continents': {'terms': {'field': 'locations.geonames_details.continent_code.raw', 'size': 10, 'min_doc_count': 1}},
'statuses': {'terms': {'field': 'status.raw', 'size': 10, 'min_doc_count': 1}}},
'track_total_hits': True, 'from': 0, 'size': 20}

def test_empty_query_default(self):
Expand Down Expand Up @@ -284,6 +284,38 @@ def test_query_advanced(self):
query = build_search_query({'query.advanced': 'query terms'})
self.assertEqual(query.to_dict(), expected)

def test_query_advanced_nfc_normalization(self):
nfd = 'locations.geonames_details.name:Hu\u0065\u0302\u0301'
nfc_form = 'locations.geonames_details.name:Hu\u1ebf'
self.assertNotEqual(nfd, nfc_form)
query = build_search_query({'query.advanced': nfd}).to_dict()
self.assertEqual(
query['query']['bool']['must'][0]['query_string']['query'], nfc_form)

def test_query_nfc_normalization(self):
query = build_search_query({'query': 'Montre\u0301al'}).to_dict()
self.assertEqual(
query['query']['bool']['must'][0]['nested']['query']['query_string']['query'],
'Montr\u00e9al')

def test_filter_nfc_normalization(self):
query = build_search_query({
'filter': 'locations.geonames_details.country_name:Cura\u0063\u0327ao',
'all_status': ''
}).to_dict()
self.assertIn(
{'terms': {'locations.geonames_details.country_name': ('Cura\u00e7ao',)}},
query['query']['bool']['filter'])

def test_aggregations_use_raw_subfields(self):
aggs = build_search_query({}).to_dict()['aggs']
self.assertEqual(aggs['types']['terms']['field'], 'types.raw')
self.assertEqual(aggs['countries']['terms']['field'],
'locations.geonames_details.country_code.raw')
self.assertEqual(aggs['continents']['terms']['field'],
'locations.geonames_details.continent_code.raw')
self.assertEqual(aggs['statuses']['terms']['field'], 'status.raw')

def test_query_advanced_all_status(self):
expected = {'query': {
'bool': {
Expand Down
79 changes: 62 additions & 17 deletions rorapi/v2/index_template_es7.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,15 @@
"type": "asciifolding",
"preserve_original": true
}
},
"normalizer": {
"folding_normalizer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding"
]
}
}
}
},
Expand Down Expand Up @@ -61,7 +70,8 @@
"type": "keyword"
},
"type": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"preferred": {
"type": "keyword"
Expand All @@ -78,7 +88,8 @@
"analyzer": "simple"
},
"type": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
}
}
},
Expand All @@ -90,22 +101,38 @@
"geonames_details": {
"properties": {
"continent_code": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer",
"fields": {
"raw": {
"type": "keyword"
}
}
},
"continent_name": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"country_code": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer",
"fields": {
"raw": {
"type": "keyword"
}
}
},
"country_name": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"country_subdivision_code": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"country_subdivision_name": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"lat": {
"type": "float"
Expand All @@ -114,7 +141,8 @@
"type": "float"
},
"name": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
}
}
}
Expand All @@ -133,23 +161,33 @@
"analyzer": "string_lowercase",
"fielddata": true
}
}
},
"analyzer": "string_lowercase"
},
"lang": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"types": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
}
}
},
"types": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer",
"fields": {
"raw": {
"type": "keyword"
}
}
},
"relationships": {
"properties": {
"type": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer"
},
"label": {
"type": "text",
Expand All @@ -162,15 +200,22 @@
"analyzer": "string_lowercase",
"fielddata": true
}
}
},
"analyzer": "string_lowercase"
},
"id": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword"
"type": "keyword",
"normalizer": "folding_normalizer",
"fields": {
"raw": {
"type": "keyword"
}
}
},
"names_ids": {
"type": "nested",
Expand Down Expand Up @@ -223,4 +268,4 @@
}
}
}
}
}
Loading