fix(trees): improve tree node search performance

specifyweb/backend/stored_queries/execution.py

@@ -1071,6 +1071,17 @@ def build_query(
     order_by_exprs = []
     selected_fields = []
     predicates_by_field = defaultdict(list)
+    # Materializing a subtree can cost more than parent lookups when the caller
+    # has already restricted the query to explicit record IDs.
+    optimize_tree = props.recordsetid is None and not any(
+        len(fs.fieldspec.join_path) == 1
+        and base_table is not None
+        and fs.fieldspec.get_field() == base_table.idField
+        and fs.op_num in (1, 10)
+        and not fs.negate
+        and fs.value not in ('', None)
+        for fs in field_specs
+    )
     # augment_field_specs(field_specs, formatauditobjs)
     for fs in field_specs:
         # sort_type = SORT_TYPES[fs.sort_type]
@@ -1082,7 +1093,8 @@ def build_query(
             continue
 
         query, field, predicate = fs.add_to_query(
-            query, formatauditobjs=props.formatauditobjs, collection=collection, user=user
+            query, formatauditobjs=props.formatauditobjs, collection=collection, user=user,
+            optimize_tree=optimize_tree,
         )
 
         if field is None:

specifyweb/backend/stored_queries/query_construct.py

@@ -2,6 +2,7 @@
 from collections import namedtuple, deque
 
 from sqlalchemy import orm, sql, or_
+from django.db.models import F, Q
 
 import specifyweb.specify.models as spmodels
 from specifyweb.backend.trees.utils import get_treedefs
@@ -11,12 +12,6 @@
 
 logger = logging.getLogger(__name__)
 
-def _safe_filter(query):
-    count = query.count()
-    if count <= 1:
-        return query.first()
-    raise Exception(f"Got more than one matching: {list(query)}")
-
 class QueryConstruct(namedtuple('QueryConstruct', 'collection objectformatter query join_cache tree_rank_count internal_filters')):
 
     def __new__(cls, *args, **kwargs):
@@ -27,7 +22,58 @@ def __new__(cls, *args, **kwargs):
         kwargs['internal_filters'] = []
         return super().__new__(cls, *args, **kwargs)
 
-    def handle_tree_field(self, node, table, tree_rank: TreeRankQuery, next_join_path, current_field_spec: QueryFieldSpec):
+    def tree_rank_metadata(self, table, tree_rank):
+        """Resolve ranks once per query, shared by all paths into the same tree."""
+        query = self
+        defs_key = ('TreeDefinitions', table.name)
+        if defs_key not in query.join_cache:
+            query = query._replace(join_cache=query.join_cache.copy())
+            query.join_cache[defs_key] = get_treedefs(query.collection, table.name)
+        treedefs = query.join_cache[defs_key]
+
+        # TreeRankQuery equality does not include the explicit tree definition.
+        rank_key = ('TreeRankItems', table.name, tree_rank.name, tree_rank.treedef_id)
+        if rank_key not in query.join_cache:
+            item_model = getattr(spmodels, table.django_name + 'treedefitem')
+            def_ids = [
+                def_id for def_id, _ in treedefs
+                if tree_rank.treedef_id is None or tree_rank.treedef_id == def_id
+            ]
+            items = item_model.objects.filter(
+                treedef_id__in=def_ids, name=tree_rank.name
+            ).values_list('treedef_id', 'id')
+            by_definition = {}
+            for def_id, item_id in items:
+                if def_id in by_definition:
+                    raise Exception('Got more than one matching tree rank')
+                by_definition[def_id] = item_id
+            ranks = [(def_id, by_definition[def_id]) for def_id in def_ids if def_id in by_definition]
+            query = query._replace(join_cache=query.join_cache.copy())
+            query.join_cache[rank_key] = ranks
+        return query, treedefs, query.join_cache[rank_key]
+
+    def tree_numbering_available(self, table, treedefs):
+        """Fall back to parent links for trees with missing or reversed intervals.
+
+        Tree writes maintain the nesting invariant. Do not renumber a tree while
+        reading it, or cache this check across requests (uploads can change it).
+        """
+        cache_key = ('TreeNumbering', table.name)
+        query = self
+        if cache_key not in query.join_cache:
+            model = getattr(spmodels, table.django_name)
+            invalid = model.objects.filter(
+                definition_id__in=[def_id for def_id, _ in treedefs]
+            ).filter(
+                Q(nodenumber__isnull=True)
+                | Q(highestchildnodenumber__isnull=True)
+                | Q(highestchildnodenumber__lt=F('nodenumber'))
+            ).exists()
+            query = query._replace(join_cache=query.join_cache.copy())
+            query.join_cache[cache_key] = not invalid
+        return query, query.join_cache[cache_key]
+
+    def handle_tree_field(self, node, table, tree_rank: TreeRankQuery, next_join_path, current_field_spec: QueryFieldSpec, use_range=False, use_rank_lookup=False):
         query = self
         query_before_tree_joins = query
         if query.collection is None:  # Not sure it makes sense to query across collections
@@ -40,13 +86,84 @@ def handle_tree_field(self, node, table, tree_rank: TreeRankQuery, next_join_pat
         treedefitem_column = table.name + 'TreeDefItemID'
         treedef_column = table.name + 'TreeDefID'
 
+        query, treedefs, treedefs_with_ranks = query.tree_rank_metadata(table, tree_rank)
+
+        if not treedefs_with_ranks:
+            logger.warning(
+                "Didn't find tree rank %r across any %s tree; skipping field",
+                tree_rank.name,
+                table.name,
+            )
+            return query_before_tree_joins, None, None, table
+
+        if use_range:
+            query, use_range = query.tree_numbering_available(table, treedefs)
+        if use_range:
+            # One matching ancestor at this rank, including the node itself.
+            # Keep the filter on the ancestor column so the optimizer can start
+            # with selective ID/name indexes and range-scan descendants.
+            range_cache_key = (node, 'TreeRankLookup' if use_rank_lookup else 'TreeRankRange',
+                               tree_rank.name, tree_rank.treedef_id)
+            if range_cache_key in query.join_cache:
+                ancestor = query.join_cache[range_cache_key]
+            else:
+                ancestor = orm.aliased(getattr(models, table.name))
+                if use_rank_lookup:
+                    # Map descendants to their ancestor once for this rank. The
+                    # unfiltered mapping preserves displayed values when filters
+                    # are combined with OR, while the PK join avoids a range
+                    # comparison against every node at the requested rank.
+                    model = getattr(models, table.name)
+                    rank_node = orm.aliased(model)
+                    child = orm.aliased(model)
+                    lookup = sql.select(
+                        rank_node._id.label('node_id'),
+                        rank_node._id.label('ancestor_id'),
+                        getattr(rank_node, treedef_column).label('definition_id'),
+                    ).where(
+                        getattr(rank_node, treedefitem_column).in_(
+                            [item_id for _, item_id in treedefs_with_ranks]
+                        ),
+                    ).cte(recursive=True)
+                    lookup = lookup.union(sql.select(
+                        child._id, lookup.c.ancestor_id, getattr(child, treedef_column),
+                    ).join(lookup, sql.and_(
+                        child.ParentID == lookup.c.node_id,
+                        getattr(child, treedef_column) == lookup.c.definition_id,
+                    )))
+                    query = query._replace(query=query.query.outerjoin(
+                        lookup, node._id == lookup.c.node_id,
+                    ).outerjoin(ancestor, ancestor._id == lookup.c.ancestor_id))
+                else:
+                    query = query._replace(query=query.query.outerjoin(ancestor, sql.and_(
+                        getattr(node, treedef_column) == getattr(ancestor, treedef_column),
+                        getattr(ancestor, treedefitem_column).in_(
+                            [item_id for _, item_id in treedefs_with_ranks]
+                        ),
+                        node.nodeNumber.between(ancestor.nodeNumber, ancestor.highestChildNodeNumber),
+                    )))
+                query = query._replace(join_cache=query.join_cache.copy())
+                query.join_cache[range_cache_key] = ancestor
+            field_spec = current_field_spec._replace(
+                root_table=table,
+                root_sql_table=ancestor,
+                join_path=next_join_path,
+            )
+            query, column, field, result_table = field_spec.add_spec_to_query(query)
+            query = query._replace(internal_filters=[
+                *query.internal_filters,
+                or_(
+                    getattr(node, treedef_column).in_([def_id for def_id, _ in treedefs_with_ranks]),
+                    getattr(node, treedef_column).is_(None),
+                ),
+            ])
+            return query, column, field, result_table
+
         cache_key = (node, 'TreeRanks')
         if cache_key in query.join_cache:
             logger.debug("using join cache for %r tree ranks.", node)
             ancestors, treedefs = query.join_cache[cache_key]
         else:
-            treedefs = get_treedefs(query.collection, table.name)
-
             # We need to take the max here. Otherwise, it is possible that the same rank
             # name may not occur at the same level across tree defs.
             max_depth = max(depth for _, depth in treedefs)
@@ -61,24 +178,6 @@ def handle_tree_field(self, node, table, tree_rank: TreeRankQuery, next_join_pat
             query = query._replace(join_cache=query.join_cache.copy())
             query.join_cache[cache_key] = (ancestors, treedefs)
 
-        item_model = getattr(spmodels, table.django_name + "treedefitem")
-
-        # TODO: optimize out the ranks that appear? cache them
-        treedefs_with_ranks: list[tuple[int, int]] = [tup for tup in [
-            (treedef_id, _safe_filter(item_model.objects.filter(treedef_id=treedef_id, name=tree_rank.name).values_list('id', flat=True)))
-            for treedef_id, _ in treedefs
-            # For constructing tree queries for batch edit
-            if (tree_rank.treedef_id is None or tree_rank.treedef_id == treedef_id)
-            ] if tup[1] is not None]
-
-        if not treedefs_with_ranks:
-            logger.warning(
-                "Didn't find tree rank %r across any %s tree; skipping field",
-                tree_rank.name,
-                table.name,
-            )
-            return query_before_tree_joins, None, None, table
-
         treedefitem_params = [treedefitem_id for (_, treedefitem_id) in treedefs_with_ranks]
 
         def make_tree_field_spec(tree_node):

specifyweb/backend/stored_queries/queryfield.py

@@ -1,13 +1,16 @@
+from __future__ import annotations
+
 from email.policy import strict
 import logging
 from collections import namedtuple
-from typing import Any, NamedTuple, Literal
+from typing import Any, NamedTuple, Literal, TYPE_CHECKING
 
 from .query_ops import QueryOps, QUERYFIELD_OPERATION_NUMBER
-from .queryfieldspec import QueryFieldSpec
 
-logger = logging.getLogger(__name__)
+if TYPE_CHECKING:
+    from .queryfieldspec import QueryFieldSpec
 
+logger = logging.getLogger(__name__)
 
 QUREYFIELD_SORT_T = Literal[
     0,  # NONE
@@ -59,6 +62,8 @@
 
     @classmethod
     def from_spqueryfield(cls, field: EphemeralField, value: str | None=None):
+        from .queryfieldspec import QueryFieldSpec
+
         logger.info("processing field from %r", field)
         fieldspec = QueryFieldSpec.from_stringid(
             field.stringId, field.isRelFld)
@@ -78,7 +83,9 @@
             strict=field.isStrict,
         )
 
-    def add_to_query(self, query, no_filter=False, formatauditobjs=False, collection=None, user=None):
+    def add_to_query(self, query, no_filter=False, formatauditobjs=False, collection=None, user=None, optimize_tree=True):
+        from .queryfieldspec import TreeRankQuery
+
         logger.info("adding field %s", self)
         value_required_for_filter = QueryOps.OPERATIONS[self.op_num] not in (
             "op_true",  # 6
@@ -92,6 +99,28 @@
             self.value == "" and value_required_for_filter and not self.negate
         )
 
+        # Positive scalar filters reject missing ancestors, allowing MariaDB to
+        # start at the matching rank instead of walking up from every specimen.
+        # Empty/negated filters need the existing missing-ancestor semantics;
+        # relationship paths and date transformations retain their usual joins.
+        path = self.fieldspec.join_path
+        use_tree_range = (
+            optimize_tree
+            and not no_filter
+            and not self.negate
+            and (not value_required_for_filter or isinstance(self.value, str))
+            and self.op_num in {0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 15, 18}
+            and len(path) >= 2
+            and isinstance(path[-2], TreeRankQuery)
+            and not path[-1].is_relationship
+            and sum(isinstance(part, TreeRankQuery) for part in path) == 1
+            and self.fieldspec.date_part is None
+        )
+        # Exact matches can start at the matching ancestor and range over its
+        # descendants. Other operators share a descendant-to-rank lookup so
+        # they do not repeatedly walk every node's parent chain.
+        use_rank_lookup = use_tree_range and self.op_num != 1
+
         return self.fieldspec.add_to_query(
             query,
             value=self.value,
@@ -102,4 +131,6 @@
             strict=self.strict,
             collection=collection,
             user=user,
+            use_tree_range=use_tree_range,
+            use_rank_lookup=use_rank_lookup,
         )

specifyweb/backend/stored_queries/queryfieldspec.py

@@ -526,14 +526,18 @@ def add_to_query(
         strict=False,
         collection=None,
         user=None,
+        use_tree_range=False,
+        use_rank_lookup=False,
     ):
         # print "############################################################################"
         # print "formatauditobjs " + str(formatauditobjs)
         # if self.get_field() is not None:
         #    print "field name " + self.get_field().name
         # print "is auditlog obj format field = " + str(self.is_auditlog_obj_format_field(formatauditobjs))
         # print "############################################################################"
-        query, orm_field, field, table = self.add_spec_to_query(query, formatter)
+        query, orm_field, field, table = self.add_spec_to_query(
+            query, formatter, use_tree_range=use_tree_range, use_rank_lookup=use_rank_lookup
+        )
         if orm_field is None:
             return query, None, None
         return self.apply_filter(
@@ -551,7 +555,7 @@ def add_to_query(
         )
 
     def add_spec_to_query(
-        self, query, formatter=None, aggregator=None, cycle_detector=[]
+        self, query, formatter=None, aggregator=None, cycle_detector=[], use_tree_range=False, use_rank_lookup=False
     ):
 
         if self.get_field() is None:
@@ -589,6 +593,8 @@ def add_spec_to_query(
                     field,
                     self.join_path[tree_rank_idx + 1 :],
                     self,
+                    use_range=use_tree_range,
+                    use_rank_lookup=use_rank_lookup,
                 )
             else:
                 try: