Bugfix/9463 improve vocab related searchable fields

pom.xml

@@ -192,7 +192,7 @@
             <dependency>
                 <groupId>au.org.aodn</groupId>
                 <artifactId>stacmodel</artifactId>
-                <version>1.0.3</version>
+                <version>1.0.4</version>
             </dependency>
         </dependencies>
     </dependencyManagement>

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java

@@ -467,6 +467,33 @@ public static SortOptions getDatasetGroupPrioritySort(List<String> candidates) {
                         .build();
         }
 
+        /**
+         * Every word of the literal must appear somewhere across title, description and the vocab fields, e.g. for
+         * "coastal wave data" the word "coastal" may come from an organisation vocab, "wave" from the title and
+         * "data" from the description.
+         * cross_fields is term-centric, so the words may come from different fields. best_fields (fuzzy_title and
+         * fuzzy_desc) is field-centric, its operator AND needs all words in one field. Elastic does not allow fuzziness
+         * for cross_fields, typos are still handled by fuzzy_title and fuzzy_desc.
+         * Each word scores its best field plus 0.1 times the others, the same tie_breaker the field and synonyms
+         * multi_match queries use, applied per word across these fields.
+         * The .synonyms sub-fields use another analyzer so they would form a separate group that must match all words
+         * on its own, acronyms are left to fuzzy_title and fuzzy_desc. Credits is long free text that names many
+         * organisations, so it is left out to avoid noise.
+         */
+        public static Query getAllWordsAcrossFieldsQuery(String literal) {
+                return MultiMatchQuery.of(m -> m
+                                .type(TextQueryType.CrossFields)
+                                .fields(
+                                        StacBasicField.Title.searchField + "^2",
+                                        StacBasicField.Description.searchField,
+                                        StacBasicField.ParameterVocabs.searchField,
+                                        StacBasicField.PlatformVocabs.searchField,
+                                        StacBasicField.OrganisationVocabs.searchField)
+                                .tieBreaker(0.1)
+                                .operator(Operator.And)
+                                .query(literal))._toQuery();
+        }
+
         @Override
         public Query getPropertyEqualToQuery(String literal) {
                 if (getOverridePropertyEqualsToQuery() == null) {

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -437,25 +437,52 @@ private static List<Query> createKeywordShouldClauses(List<String> keywords) {
             // If search text with double quote, remove quotee, otherwise keeps same
             String term = isExact ? t.substring(1, t.length() - 1) : t;
 
+            List<Query> clauses = new ArrayList<>();
             if (isExact) {
                 // Match phrase in original title and description, not use fuzzy fields
-                should.add(CQLFields.title.getPropertyEqualToQuery(term));
-                should.add(CQLFields.description.getPropertyEqualToQuery(term));
+                clauses.add(CQLFields.title.getPropertyEqualToQuery(term));
+                clauses.add(CQLFields.description.getPropertyEqualToQuery(term));
             }
             else {
-                should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term));
-                should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term));
+                clauses.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term));
+                clauses.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term));
+            }
+            clauses.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term));
+            clauses.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
+            clauses.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
+            clauses.add(CQLFields.id.getPropertyEqualToQuery(term));
+            // credit_contains is a phrase match, the same for exact and non-exact keywords
+            clauses.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
+            if (!isExact && hasSeveralWords(term)) {
+                should.add(bestOfClausesOrAllWordsAcrossFields(term, clauses));
+            }
+            else {
+                should.addAll(clauses);
             }
-            should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term));
-            should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
-            should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
-            should.add(CQLFields.id.getPropertyEqualToQuery(term));
-            // credit_contains uses match query by default, exact match is not applied here
-            should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
         }
         return should;
     }
 
+    /**
+     * For a keyword whose words may be spread across fields, e.g. "coastal wave data" where "coastal" is only in an
+     * organisation vocab. A record matches if either the keyword's clauses or the all words across fields query
+     * matches, and keeps the higher of the two scores. The clauses are summed in a bool exactly as when they are added
+     * one by one, so a record never scores below what the clauses alone give, and each clause is evaluated once.
+     */
+    private static Query bestOfClausesOrAllWordsAcrossFields(String term, List<Query> clauses) {
+        Query keywordClauses = BoolQuery.of(b -> b.should(clauses))._toQuery();
+        return DisMaxQuery.of(d -> d
+                .queries(keywordClauses, CQLFields.getAllWordsAcrossFieldsQuery(term)))._toQuery();
+    }
+
+    /**
+     * A single word found in title, description or a vocab is already matched by fuzzy_title, fuzzy_desc or the vocab
+     * phrase clause, so matching it across fields cannot find more records.
+     */
+    private static boolean hasSeveralWords(String term) {
+        return term.trim().split("\\s+").length > 1;
+    }
+
     /**
      * Dataset-group candidate values for the priority sort, expanded from the search keywords with the same exact/quoted handling as the should clauses.
      */

server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java

@@ -199,6 +199,33 @@ public void explainSimplifiedFormatReportsMultiWordMatches() throws IOException
         }
     }
 
+    @Test
+    public void explainSimplifiedFormatReportsWordsAcrossFields() throws IOException {
+        // 'queensland' is only in an organisation vocab and 'current velocity' is in the title, so only the
+        // all words across fields fallback matches, and each word is reported against the field it came from
+        insertRecordsWithExplicitIds("ae86e2f5-eaaf-459e-a405-e654d85adb9c.json");
+
+        URI simpleUri = explainUri("q", "queensland current velocity", "format", "simple");
+
+        ResponseEntity<JsonNode> response = testRestTemplate.getForEntity(simpleUri, JsonNode.class);
+
+        assertTrue(response.getStatusCode().is2xxSuccessful());
+        JsonNode hit = Objects.requireNonNull(response.getBody()).path("hits").path(0);
+        assertEquals("ae86e2f5-eaaf-459e-a405-e654d85adb9c", hit.path("id").asText());
+        assertTrue(hit.path("es_relevance").asDouble() > 0.0, "the fallback scores the record");
+
+        List<String> reported = StreamSupport.stream(hit.path("matched_terms").spliterator(), false)
+                .map(term -> term.path("field").asText() + ":" + term.path("term").asText())
+                .toList();
+        assertTrue(reported.contains("summaries.organisation_vocabs:queensland"),
+                "'queensland' must be reported for the organisation vocab, got: " + reported);
+        // the best of the fields is reported, both words are in the title and may be in the description too
+        for (String word : List.of("current", "velocity")) {
+            assertTrue(reported.contains("title:" + word) || reported.contains("description:" + word),
+                    "'" + word + "' must be reported, got: " + reported);
+        }
+    }
+
     @Test
     public void explainSimplifiedFormatReportsTheDatasetGroupPriorityInTheSortValues() throws IOException {
         // the record sits in the "imos" dataset group and its credits name IMOS, so the search recalls it on the text and the group then ranks it

server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java

@@ -133,6 +133,36 @@ public void verifyApiCollectionsQueryOnText2() throws IOException {
                 collections.getBody().getCollections().get(1).getId(),
                 "Correct UUID - 9fdb1eee-bc28-43a9-88c5-972324784837");
     }
+    /**
+     * The words of one keyword may be spread across title, description and the vocab fields.
+     * ae86e2f5-eaaf-459e-a405-e654d85adb9c - 'queensland' is only in an organisation vocab, 'current velocity' is in the title
+     * 9fdb1eee-bc28-43a9-88c5-972324784837 - has none of the words
+     *
+     * @throws IOException - IO Exception
+     */
+    @Test
+    public void verifyApiCollectionsQueryWordsAcrossFields() throws IOException {
+        super.insertJsonToElasticRecordIndex(
+                "ae86e2f5-eaaf-459e-a405-e654d85adb9c.json",
+                "9fdb1eee-bc28-43a9-88c5-972324784837.json"
+        );
+
+        // No single field holds all words, only the all words across fields fallback finds it
+        ResponseEntity<ExtendedCollections> collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q=queensland current velocity", ExtendedCollections.class);
+        assertEquals(1, Objects.requireNonNull(collections.getBody()).getTotal(), "Only 1 hit");
+        assertEquals(
+                "ae86e2f5-eaaf-459e-a405-e654d85adb9c",
+                collections.getBody().getCollections().get(0).getId(),
+                "Correct UUID - ae86e2f5-eaaf-459e-a405-e654d85adb9c");
+
+        // Every word must be found somewhere, 'zebra' is in no field
+        collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q=queensland current zebra", ExtendedCollections.class);
+        assertEquals(0, Objects.requireNonNull(collections.getBody()).getTotal(), "No hit when a word is missing");
+
+        // A quoted keyword is an exact phrase search, the fallback does not apply
+        collections = testRestTemplate.getForEntity(getBasePath() + "/collections?q=\"queensland current velocity\"", ExtendedCollections.class);
+        assertEquals(0, Objects.requireNonNull(collections.getBody()).getTotal(), "No hit for a quoted keyword");
+    }
     /**
      * The datetime field after xxx/.. xxx/ etc. It uses CQL internally so no need to test Before After During in CQL
      */

server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java

@@ -122,6 +122,23 @@ public void disMaxReportsHighestChildWhenNoChildEqualsParent() {
         assertEquals("title", terms.get(0).getField());
     }
 
+    @Test
+    public void disMaxOfKeywordClausesAndAcrossFieldsReportsOnlyTheWinningGroup() {
+        // a multi word keyword: dis_max of the summed keyword clauses and the cross_fields match, each a "sum of:" group
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = collectDisMax("max of:", 12.0f,
+                detail(8.0f, "sum of:",
+                        detail(5.0f, "weight(description:wave in 1) [BM25], result of:"),
+                        detail(3.0f, "weight(description:data in 1) [BM25], result of:")),
+                detail(12.0f, "sum of:",
+                        detail(9.0f, "weight(title:wave in 1) [BM25], result of:"),
+                        detail(2.0f, "weight(summaries.organisation_vocabs:coastal in 1) [BM25], result of:"),
+                        detail(1.0f, "weight(description:data in 1) [BM25], result of:")));
+
+        assertEquals(List.of("title:wave", "summaries.organisation_vocabs:coastal", "description:data"),
+                terms.stream().map(term -> term.getField() + ":" + term.getTerm()).toList(),
+                "only the terms of the higher scoring group are reported");
+    }
+
     @Test
     public void disMaxWithTieBreakerReportsOnlyTheWinner() {
         // best_fields multi_match with tie_breaker 0.1: 36.3 + 0.1 * 12.1