Exclude acronyms from multisearch candidate retrieval

rorapi/common/es_utils.py

@@ -70,6 +70,47 @@ def add_common_query(self, fields, terms):
             ],
         )
 
+    def add_phrase_query_affiliation_names(self, terms):
+        """Phrase match on acronym-free names in ``affiliation_match.names``."""
+        self.search.query = Q(
+            "nested",
+            path="affiliation_match.names",
+            score_mode="max",
+            query=Q("match_phrase", **{"affiliation_match.names.name": terms}),
+        )
+
+    def add_common_query_affiliation_names(self, terms):
+        self.search.query = Q(
+            "nested",
+            path="affiliation_match.names",
+            score_mode="max",
+            query=Q(
+                "common",
+                **{
+                    "affiliation_match.names.name": {
+                        "query": terms,
+                        "cutoff_frequency": 0.001,
+                    }
+                },
+            ),
+        )
+
+    def add_fuzzy_query_affiliation_names(self, terms):
+        self.search.query = Q(
+            "nested",
+            path="affiliation_match.names",
+            score_mode="max",
+            query=Q(
+                "match",
+                **{
+                    "affiliation_match.names.name": {
+                        "query": terms,
+                        "fuzziness": "AUTO",
+                    }
+                },
+            ),
+        )
+
     def add_match_query(self, terms):
         self.search = self.search.query("match", acronyms=terms)
 

rorapi/common/index_helpers.py

@@ -53,6 +53,9 @@ def enrich_org_for_index(org):
         'id': n
     } for n in get_nested_ids_v2(org)]
     org['affiliation_match'] = get_affiliation_match_doc(org)
+    org['acronyms'] = [
+        n["value"] for n in org["names"] if "acronym" in n["types"]
+    ]
     return org
 
 

rorapi/common/matching.py

@@ -179,7 +179,6 @@ def match_by_query(text, matching_type, query, countries):
 def match_by_type(text, matching_type, countries):
     """Match affiliation text using specific matching mode/type."""
 
-    fields = ["names.value.norm"]
     substrings = []
     if matching_type == MATCHING_TYPE_HEURISTICS:
         h1 = re.search(r"University of ([^\s]+)", text)
@@ -204,17 +203,18 @@ def match_by_type(text, matching_type, countries):
 
     queries = [ESQueryBuilder() for _ in substrings]
 
+    normalized = normalize(text)
     for s, q in zip(substrings, queries):
         if matching_type == MATCHING_TYPE_PHRASE:
-            q.add_phrase_query(fields, normalize(text))
+            q.add_phrase_query_affiliation_names(normalized)
         elif matching_type == MATCHING_TYPE_COMMON:
-            q.add_common_query(fields, normalize(text))
+            q.add_common_query_affiliation_names(normalized)
         elif matching_type == MATCHING_TYPE_FUZZY:
-            q.add_fuzzy_query(fields, normalize(text))
+            q.add_fuzzy_query_affiliation_names(normalized)
         elif matching_type == MATCHING_TYPE_ACRONYM:
-            q.add_match_query(normalize(text))
+            q.add_match_query(normalized)
         elif matching_type == MATCHING_TYPE_HEURISTICS:
-            q.add_common_query(fields, normalize(text))
+            q.add_common_query_affiliation_names(normalized)
     queries = [q.get_query() for q in queries]
     matched = [
         match_by_query(t, matching_type, q, countries)

rorapi/tests/tests_integration/tests_matching_v2.py

@@ -41,3 +41,21 @@ def test_query_organizations(self):
             self.assertTrue(
                 i.get('matching_type') in
                 ['PHRASE', 'ACRONYM', 'FUZZY', 'HEURISTICS', 'COMMON TERMS', 'EXACT'])
+
+
+class MultisearchAcronymExclusionTestCase(SimpleTestCase):
+    """ror-roadmap#345: multisearch must not choose orgs via acronym-only names."""
+
+    def test_ucla_does_not_choose_wrong_org(self):
+        output = requests.get(BASE_URL, {
+            'affiliation': 'UCLA',
+            'multisearch': '',
+        }).json()
+        chosen = [i for i in output.get('items', []) if i.get('chosen')]
+        for item in chosen:
+            self.assertNotEqual(
+                item['organization']['id'],
+                'https://ror.org/03qgg3111',
+                'Universidad Centroccidental Lisandro Alvarado must not be chosen '
+                'for bare UCLA via PHRASE on acronym',
+            )

rorapi/tests/tests_unit/tests_es_utils_v2.py

@@ -161,6 +161,72 @@ def test_fuzzy_query(self):
                 'track_total_hits': True
             })
 
+    def test_phrase_query_affiliation_names(self):
+        qb = ESQueryBuilder()
+        qb.add_phrase_query_affiliation_names('query terms')
+
+        self.assertEqual(
+            qb.get_query().to_dict(), {
+                'query': {
+                    'nested': {
+                        'path': 'affiliation_match.names',
+                        'score_mode': 'max',
+                        'query': {
+                            'match_phrase': {
+                                'affiliation_match.names.name': 'query terms'
+                            }
+                        }
+                    }
+                },
+                'track_total_hits': True
+            })
+
+    def test_common_query_affiliation_names(self):
+        qb = ESQueryBuilder()
+        qb.add_common_query_affiliation_names('query terms')
+
+        self.assertEqual(
+            qb.get_query().to_dict(), {
+                'query': {
+                    'nested': {
+                        'path': 'affiliation_match.names',
+                        'score_mode': 'max',
+                        'query': {
+                            'common': {
+                                'affiliation_match.names.name': {
+                                    'query': 'query terms',
+                                    'cutoff_frequency': 0.001
+                                }
+                            }
+                        }
+                    }
+                },
+                'track_total_hits': True
+            })
+
+    def test_fuzzy_query_affiliation_names(self):
+        qb = ESQueryBuilder()
+        qb.add_fuzzy_query_affiliation_names('query terms')
+
+        self.assertEqual(
+            qb.get_query().to_dict(), {
+                'query': {
+                    'nested': {
+                        'path': 'affiliation_match.names',
+                        'score_mode': 'max',
+                        'query': {
+                            'match': {
+                                'affiliation_match.names.name': {
+                                    'query': 'query terms',
+                                    'fuzziness': 'AUTO'
+                                }
+                            }
+                        }
+                    }
+                },
+                'track_total_hits': True
+            })
+
     def test_add_filters(self):
         qb = ESQueryBuilder()
         qb.add_match_all_query()

rorapi/tests/tests_unit/tests_index_helpers.py

@@ -73,6 +73,7 @@ def test_enrich_org_for_index(self):
         self.assertIn({'name': 'University of Example'}, org['names_ids'])
         self.assertIn({'id': '01an7q238'}, org['names_ids'])
         self.assertEqual(org['affiliation_match']['primary'], 'University of Example')
+        self.assertEqual(org['acronyms'], ['UoE'])
 
 
 class BulkIndexWithBackupTestCase(SimpleTestCase):

rorapi/v2/index_template_es7.json

@@ -61,6 +61,10 @@
                 "type": "text",
                 "analyzer": "simple"
             },
+            "acronyms": {
+                "type": "keyword",
+                "normalizer": "folding_normalizer"
+            },
             "established": {
                 "type": "date"
             },