diff --git a/rorapi/common/queries.py b/rorapi/common/queries.py index e3c4a4a5..1577c0e0 100644 --- a/rorapi/common/queries.py +++ b/rorapi/common/queries.py @@ -1,5 +1,6 @@ import re import json +import unicodedata from titlecase import titlecase from collections import defaultdict @@ -187,6 +188,14 @@ def validate(params): return Errors(errors) if errors else None +def nfc(value): + """Normalizes a string to Unicode NFC so decomposed (NFD) input matches + the precomposed characters indexed in Elasticsearch""" + if isinstance(value, str): + return unicodedata.normalize("NFC", value) + return value + + def build_search_query(params): """Builds search query from API parameters""" @@ -198,20 +207,21 @@ def build_search_query(params): del params["all_status"] if "query.advanced" in params: - qb.add_string_query_advanced(params.get("query.advanced")) + qb.add_string_query_advanced(nfc(params.get("query.advanced"))) elif "query" in params: - ror_id = get_ror_id(params.get("query")) + query = nfc(params.get("query")) + ror_id = get_ror_id(query) if ror_id is not None: qb.add_id_query(ror_id) else: - qb.add_string_query(params.get("query")) + qb.add_string_query(query) else: qb.add_match_all_query() if "filter" in params or (not "all_status" in params): filters = [ f.split(":") - for f in filter_string_to_list(params.get("filter", "")) + for f in filter_string_to_list(nfc(params.get("filter", ""))) if f ] # normalize filter values based on casing conventions used in ROR records @@ -247,10 +257,10 @@ def build_search_query(params): qb.add_aggregations( [ - ("types", "types"), - ("countries", "locations.geonames_details.country_code"), - ("continents", "locations.geonames_details.continent_code"), - ("statuses", "status"), + ("types", "types.raw"), + ("countries", "locations.geonames_details.country_code.raw"), + ("continents", "locations.geonames_details.continent_code.raw"), + ("statuses", "status.raw"), ] ) diff --git a/rorapi/tests/tests_integration/tests_search_v2.py b/rorapi/tests/tests_integration/tests_search_v2.py index 462841be..c8279de2 100644 --- a/rorapi/tests/tests_integration/tests_search_v2.py +++ b/rorapi/tests/tests_integration/tests_search_v2.py @@ -133,3 +133,81 @@ def test_extra_word(self): }).json() self.assertTrue(items['number_of_results'] > 0) self.assertEqual(items['items'][0]['id'], 'https://ror.org/00fbnyb24') + + +class CaseAndAccentInsensitiveTestCase(SimpleTestCase): + """Case- and diacritic-insensitive matching (ror-roadmap#175, #398). + + Requires an index created from the current index_template_es7.json. + """ + + def search(self, **params): + return requests.get(BASE_URL, params).json() + + def assert_same_results(self, variants, param='query.advanced'): + results = [self.search(**{param: v}) for v in variants] + baseline = results[0] + self.assertTrue(baseline['number_of_results'] > 0, variants[0]) + baseline_ids = {i['id'] for i in baseline['items']} + for variant, result in zip(variants[1:], results[1:]): + self.assertEqual(result['number_of_results'], + baseline['number_of_results'], variant) + self.assertEqual({i['id'] for i in result['items']}, + baseline_ids, variant) + + def test_keyword_field_case(self): + self.assert_same_results([ + 'locations.geonames_details.name:Denver', + 'locations.geonames_details.name:denver', + 'locations.geonames_details.name:DENVER', + ]) + + def test_relationship_type_case(self): + self.assert_same_results([ + 'relationships.type:child', + 'relationships.type:Child', + ]) + + def test_bare_term_case(self): + self.assert_same_results(['Stellenbosch', 'stellenbosch']) + + def test_keyword_field_diacritics(self): + self.assert_same_results([ + 'locations.geonames_details.name:Huế', + 'locations.geonames_details.name:Hue', + 'locations.geonames_details.name:hue', + ]) + self.assert_same_results([ + 'locations.geonames_details.name:Montréal', + 'locations.geonames_details.name:Montreal', + ]) + + def test_nfd_and_nfc_input(self): + nfc = 'locations.geonames_details.name:Hu\u1ebf' + nfd = 'locations.geonames_details.name:Hu\u0065\u0302\u0301' + self.assertNotEqual(nfc, nfd) + self.assert_same_results([nfc, nfd]) + + def test_text_field_diacritics(self): + self.assert_same_results([ + 'names.value:"Université de Montréal"', + 'names.value:"Universite de Montreal"', + 'names.value:"universite de montreal"', + ]) + + def test_filter_case(self): + self.assert_same_results(['Denver'], param='query') + upper = requests.get(BASE_URL, {'filter': 'types:Education'}).json() + lower = requests.get(BASE_URL, {'filter': 'types:education'}).json() + self.assertTrue(upper['number_of_results'] > 0) + self.assertEqual(upper['number_of_results'], + lower['number_of_results']) + + def test_aggregation_keys_keep_original_casing(self): + # Bucket titles are derived from the raw (un-normalized) keys, e.g. the + # country name is looked up from the upper-case ISO code. + meta = self.search(**{'query.advanced': 'locations.geonames_details.name:denver'})['meta'] + countries = {b['id']: b['title'] for b in meta['countries']} + self.assertEqual(countries.get('us'), 'United States') + continents = {b['id']: b['title'] for b in meta['continents']} + self.assertEqual(continents.get('na'), 'North America') diff --git a/rorapi/tests/tests_unit/tests_queries_v2.py b/rorapi/tests/tests_unit/tests_queries_v2.py index 130ff931..251b5424 100644 --- a/rorapi/tests/tests_unit/tests_queries_v2.py +++ b/rorapi/tests/tests_unit/tests_queries_v2.py @@ -168,10 +168,10 @@ class BuildSearchQueryTestCase(SimpleTestCase): def setUp(self): self.default_query = \ - {'aggs': {'types': {'terms': {'field': 'types', 'size': 10, 'min_doc_count': 1}}, - 'countries': {'terms': {'field': 'locations.geonames_details.country_code', 'size': 10, 'min_doc_count': 1}}, - 'continents': {'terms': {'field': 'locations.geonames_details.continent_code', 'size': 10, 'min_doc_count': 1}}, - 'statuses': {'terms': {'field': 'status', 'size': 10, 'min_doc_count': 1}}}, + {'aggs': {'types': {'terms': {'field': 'types.raw', 'size': 10, 'min_doc_count': 1}}, + 'countries': {'terms': {'field': 'locations.geonames_details.country_code.raw', 'size': 10, 'min_doc_count': 1}}, + 'continents': {'terms': {'field': 'locations.geonames_details.continent_code.raw', 'size': 10, 'min_doc_count': 1}}, + 'statuses': {'terms': {'field': 'status.raw', 'size': 10, 'min_doc_count': 1}}}, 'track_total_hits': True, 'from': 0, 'size': 20} def test_empty_query_default(self): @@ -284,6 +284,38 @@ def test_query_advanced(self): query = build_search_query({'query.advanced': 'query terms'}) self.assertEqual(query.to_dict(), expected) + def test_query_advanced_nfc_normalization(self): + nfd = 'locations.geonames_details.name:Hu\u0065\u0302\u0301' + nfc_form = 'locations.geonames_details.name:Hu\u1ebf' + self.assertNotEqual(nfd, nfc_form) + query = build_search_query({'query.advanced': nfd}).to_dict() + self.assertEqual( + query['query']['bool']['must'][0]['query_string']['query'], nfc_form) + + def test_query_nfc_normalization(self): + query = build_search_query({'query': 'Montre\u0301al'}).to_dict() + self.assertEqual( + query['query']['bool']['must'][0]['nested']['query']['query_string']['query'], + 'Montr\u00e9al') + + def test_filter_nfc_normalization(self): + query = build_search_query({ + 'filter': 'locations.geonames_details.country_name:Cura\u0063\u0327ao', + 'all_status': '' + }).to_dict() + self.assertIn( + {'terms': {'locations.geonames_details.country_name': ('Cura\u00e7ao',)}}, + query['query']['bool']['filter']) + + def test_aggregations_use_raw_subfields(self): + aggs = build_search_query({}).to_dict()['aggs'] + self.assertEqual(aggs['types']['terms']['field'], 'types.raw') + self.assertEqual(aggs['countries']['terms']['field'], + 'locations.geonames_details.country_code.raw') + self.assertEqual(aggs['continents']['terms']['field'], + 'locations.geonames_details.continent_code.raw') + self.assertEqual(aggs['statuses']['terms']['field'], 'status.raw') + def test_query_advanced_all_status(self): expected = {'query': { 'bool': { diff --git a/rorapi/v2/index_template_es7.json b/rorapi/v2/index_template_es7.json index a16c10db..152ab18d 100644 --- a/rorapi/v2/index_template_es7.json +++ b/rorapi/v2/index_template_es7.json @@ -19,6 +19,15 @@ "type": "asciifolding", "preserve_original": true } + }, + "normalizer": { + "folding_normalizer": { + "type": "custom", + "filter": [ + "lowercase", + "asciifolding" + ] + } } } }, @@ -61,7 +70,8 @@ "type": "keyword" }, "type": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "preferred": { "type": "keyword" @@ -78,7 +88,8 @@ "analyzer": "simple" }, "type": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" } } }, @@ -90,22 +101,38 @@ "geonames_details": { "properties": { "continent_code": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer", + "fields": { + "raw": { + "type": "keyword" + } + } }, "continent_name": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "country_code": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer", + "fields": { + "raw": { + "type": "keyword" + } + } }, "country_name": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "country_subdivision_code": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "country_subdivision_name": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "lat": { "type": "float" @@ -114,7 +141,8 @@ "type": "float" }, "name": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" } } } @@ -133,23 +161,33 @@ "analyzer": "string_lowercase", "fielddata": true } - } + }, + "analyzer": "string_lowercase" }, "lang": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "types": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" } } }, "types": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer", + "fields": { + "raw": { + "type": "keyword" + } + } }, "relationships": { "properties": { "type": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer" }, "label": { "type": "text", @@ -162,7 +200,8 @@ "analyzer": "string_lowercase", "fielddata": true } - } + }, + "analyzer": "string_lowercase" }, "id": { "type": "keyword" @@ -170,7 +209,13 @@ } }, "status": { - "type": "keyword" + "type": "keyword", + "normalizer": "folding_normalizer", + "fields": { + "raw": { + "type": "keyword" + } + } }, "names_ids": { "type": "nested", @@ -223,4 +268,4 @@ } } } -} \ No newline at end of file +}