Dev

.github/codecov.yml

@@ -0,0 +1,22 @@
+coverage:
+  status:
+    project:
+      default:
+        # Overall coverage may not drop by more than the threshold.
+        target: auto
+        threshold: 1%
+    patch:
+      default:
+        # New and modified lines in a PR must be 80% covered.
+        target: 80%
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: false
+
+ignore:
+  - "tests/**"
+  - "docs/**"
+  - "build/**"
+  - "site/**"

src/DataBUS/AeDNAAssay.py

@@ -0,0 +1,38 @@
+from .AeDNAEntity import AeDNAEntity
+
+AEDNAASSAY_PARAMS = [
+    "datasetid",
+    "assaytypeid",
+    "assayname",
+    "targettaxonomicassay",
+    "targetgene",
+    "subfragment",
+    "ampliconsize",
+    "pcrprimerforward",
+    "pcrprimerreverse",
+    "pcrprimernameforward",
+    "pcrprimernamereverse",
+    "pcrprimerreferenceforward",
+    "pcrprimerreferencereverse",
+    "probeseq",
+    "proberef",
+    "probereporter",
+    "probequencher",
+    "probeconc",
+]
+
+
+class AeDNAAssay(AeDNAEntity):
+    """An aeDNA assay record (ndb.aednaassays).
+
+    Holds the PCR/assay configuration for a dataset: assay name and type,
+    target gene/taxon, primer sequences and probe details.
+    """
+
+    _TABLE = "ndb.aednaassays"
+    _PARAMS = AEDNAASSAY_PARAMS
+    _PK = "assayid"
+    _INT_FIELDS = ("datasetid", "assaytypeid", "ampliconsize")
+
+    def __str__(self):
+        return f"AeDNAAssay(assayid={self.assayid}, datasetid={self.datasetid}, assayname={self.assayname})"

src/DataBUS/AeDNAEntity.py

@@ -0,0 +1,45 @@
+from .neotomaHelpers.utils import validate_int_values, validate_str_values
+
+
+class AeDNAEntity:
+    """Base for the flat aeDNA tables (ndb.aednaassays, ndb.aednalibraries).
+
+    Subclasses declare four class attributes:
+
+    * ``_TABLE`` — fully-qualified table name.
+    * ``_PARAMS`` — ordered list of insert columns.
+    * ``_PK`` — primary-key column name (returned by the insert).
+    * ``_INT_FIELDS`` — the ``_PARAMS`` that are integer FKs/sizes; every other
+      param is validated as a string.
+
+    Construction validates each field and stores it as an attribute; ``_PK`` is
+    initialised to ``None`` and set after :meth:`insert_to_db`.
+    """
+
+    _TABLE: str = ""
+    _PARAMS: list[str] = []
+    _PK: str = ""
+    _INT_FIELDS: tuple[str, ...] = ()
+
+    def __init__(self, **fields):
+        setattr(self, self._PK, None)
+        for name in self._PARAMS:
+            value = fields.get(name)
+            if name in self._INT_FIELDS:
+                value = validate_int_values(value, name)
+            else:
+                value = validate_str_values(value, name)
+            setattr(self, name, value)
+
+    def insert_to_db(self, cur):
+        """Insert the record into ``_TABLE`` and return the new ``_PK`` value."""
+        cols = self._PARAMS
+        query = (
+            f"INSERT INTO {self._TABLE} ({', '.join(cols)}) "
+            f"VALUES ({', '.join('%(' + c + ')s' for c in cols)}) "
+            f"RETURNING {self._PK};"
+        )
+        cur.execute(query, {c: getattr(self, c) for c in cols})
+        pk = cur.fetchone()[0]
+        setattr(self, self._PK, pk)
+        return pk

src/DataBUS/AeDNALibrary.py

@@ -0,0 +1,30 @@
+from .AeDNAEntity import AeDNAEntity
+
+AEDNALIBRARY_PARAMS = [
+    "datasetid",
+    "assayid",
+    "libid",
+    "seqrunid",
+    "pcrplateid",
+    "barcodingpcrappr",
+    "platform",
+    "instrument",
+    "seqkit",
+    "libscreen",
+]
+
+
+class AeDNALibrary(AeDNAEntity):
+    """An aeDNA sequencing library record (ndb.aednalibraries).
+
+    Holds the library-preparation and sequencing metadata for a dataset's
+    assay: platform, instrument, kit, and library/run identifiers.
+    """
+
+    _TABLE = "ndb.aednalibraries"
+    _PARAMS = AEDNALIBRARY_PARAMS
+    _PK = "libraryid"
+    _INT_FIELDS = ("datasetid", "assayid")
+
+    def __str__(self):
+        return f"AeDNALibrary(libraryid={self.libraryid}, datasetid={self.datasetid}, assayid={self.assayid})"

src/DataBUS/AeDNAModel.py

@@ -1,6 +1,13 @@
 from .neotomaHelpers.utils import validate_int_values
 
-AEDNAMODEL_PARAMS = ["sequenceid", "taxonid", "model"]
+AEDNAMODEL_PARAMS = [
+    "sequenceid",
+    "taxonid",
+    "model",
+    "supersededbymodelid",
+    "publicationid",
+    "notes",
+]
 
 
 class AeDNAModel:
@@ -75,21 +82,21 @@ def insert_to_db(self, cur):
         self.modelid = cur.fetchone()[0]
         return self.modelid
 
-    def supersede_previous(self, cur, superseeds_list):
+    def supersede_previous(self, cur, supersedes_list):
         """Mark older aeDNA model entries as superseded by this model.
 
-        For each model name in superseeds_list, finds existing aednamodels rows
+        For each model name in supersedes_list, finds existing aednamodels rows
         that share this taxonid and that model name, then sets their
         supersededbymodelid to this model's modelid.
 
         Args:
             cur (psycopg2.cursor): Database cursor for executing queries.
-            superseeds_list (list[str]): Model names that this model supersedes.
+            supersedes_list (list[str]): Model names that this model supersedes.
 
         Returns:
             int: Total number of rows updated.
         """
-        if not superseeds_list or self.modelid is None:
+        if not supersedes_list or self.modelid is None:
             return 0
         update_q = """
             UPDATE ndb.aednamodels
@@ -100,7 +107,7 @@ def supersede_previous(self, cur, superseeds_list):
               AND modelid != %(new_modelid)s;
         """
         total_updated = 0
-        for old_model in superseeds_list:
+        for old_model in supersedes_list:
             cur.execute(
                 update_q,
                 {

src/DataBUS/Project.py

@@ -7,6 +7,10 @@
     "projectenddate",
 ]
 
+GRANT_PARAMS = ["grantname", "grantnumber", "dateawarded", "dateended"]
+
+INSTITUTION_PARAMS = ["institutionid", "institutionname", "institutionlocation"]
+
 
 class Project:
     """A research project record in the Neotoma database.
@@ -50,7 +54,7 @@ def insert_to_db(self, cur):
         Returns:
             int: The projectid assigned by the database.
         """
-        q = """
+        project_q = """
             INSERT INTO ndb.projects
                 (parentprojectid, projectname, projectdescription,
                  projectstartdate, projectenddate)
@@ -66,7 +70,7 @@ def insert_to_db(self, cur):
             "projectstartdate": self.projectstartdate,
             "projectenddate": self.projectenddate,
         }
-        cur.execute(q, inputs)
+        cur.execute(project_q, inputs)
         self.projectid = cur.fetchone()[0]
         return self.projectid
 
@@ -113,7 +117,7 @@ def insert_to_db(self, cur):
         Returns:
             int: The grantid assigned by the database.
         """
-        q = """
+        grant_q = """
             INSERT INTO ndb.grants (grantname, grantnumber, dateawarded, dateended)
             VALUES (%(grantname)s, %(grantnumber)s, %(dateawarded)s, %(dateended)s)
             RETURNING grantid;
@@ -124,7 +128,7 @@ def insert_to_db(self, cur):
             "dateawarded": self.dateawarded,
             "dateended": self.dateended,
         }
-        cur.execute(q, inputs)
+        cur.execute(grant_q, inputs)
         self.grantid = cur.fetchone()[0]
         return self.grantid
 
@@ -175,7 +179,7 @@ def insert_to_db(self, cur):
         Returns:
             str: The institutionid.
         """
-        q = """
+        institution_q = """
             INSERT INTO ndb.institutions (institutionid, institutionname, institutionlocation)
             VALUES (%(institutionid)s, %(institutionname)s, %(institutionlocation)s)
             ON CONFLICT (institutionid) DO UPDATE
@@ -188,17 +192,14 @@ def insert_to_db(self, cur):
             "institutionname": self.institutionname,
             "institutionlocation": self.institutionlocation,
         }
-        cur.execute(q, inputs)
+        cur.execute(institution_q, inputs)
         self.institutionid = cur.fetchone()[0]
         return self.institutionid
 
     def __str__(self):
         return f"Institution(institutionid={self.institutionid}, name={self.institutionname})"
 
 
-# ── Junction table helpers ────────────────────────────────────────────────────
-
-
 def insert_project_dataset(cur, projectid, datasetid):
     """Link a project to a dataset via ndb.projectdatasets."""
     q = """

src/DataBUS/Response.py

@@ -26,6 +26,8 @@ class Response:
         id_dict (dict): Mapping of data identifiers.
         name (dict): Name mapping dictionary.
         indices (list): List of indices.
+        matched (dict): Name/location match flags from close-record searches.
+        doublematched (bool): True when both name and location matched.
     """
 
     def __init__(self):
@@ -40,6 +42,8 @@ def __init__(self):
         self.name = {}
         self.indices = []
         self.counter = 0
+        self.matched = {}
+        self.doublematched = False
 
     @property
     def validAll(self):

src/DataBUS/__init__.py

@@ -16,6 +16,10 @@
 import logging
 import os
 
+from .AeDNAAssay import AeDNAAssay
+from .AeDNAEntity import AeDNAEntity
+from .AeDNALibrary import AeDNALibrary
+from .AeDNAModel import AeDNAModel
 from .AnalysisUnit import AnalysisUnit
 from .ChronControl import ChronControl
 from .Chronology import Chronology
@@ -30,9 +34,12 @@
 from .Geog import Geog, WrongCoordinates
 from .Hiatus import Hiatus
 from .LeadModel import LeadModel
+from .Project import Grant, Institution, Project
 from .Response import Response
 from .Sample import Sample
 from .SampleAge import SampleAge
+from .Sequence import Sequence
+from .SequenceData import SequenceData
 from .Site import Site
 from .Speleothem import ExternalSpeleothem, Speleothem
 from .UThSeries import UThSeries, insert_uraniumseriesdata

src/DataBUS/neotomaHelpers/__init__.py

@@ -18,5 +18,5 @@
 from .read_csv import read_csv, read_xlsx
 from .safe_step import safe_step
 from .template_to_dict import template_to_dict
-from .utils import convert_to_bp, retrieve_dict
+from .utils import convert_to_bool, convert_to_bp, retrieve_dict
 from .write_csv import write_csv

src/DataBUS/neotomaHelpers/pull_params.py

@@ -57,8 +57,16 @@ def _process_value_entry(param_name, val_entry, csv_template, table, add_unit_in
         clean_valor = ut.clean_column(
             val_entry.get("column"), template_rows, clean=not val_entry.get("rowwise")
         )
-    except KeyError:
-        return
+    except (KeyError, TypeError):
+        # The named column is not in the data at all. An entry carrying a
+        # `value:` constant supplies the datum directly instead — that is how
+        # sheets with a non-machine-readable header (projectMetadata and
+        # friends) are read. Without such a constant the parameter is genuinely
+        # absent, as before. Note this deliberately does not cover a column that
+        # exists but is blank: falling back there would mask missing data.
+        if val_entry.get("value") is None:
+            return
+        clean_valor = val_entry["value"]
     if not clean_valor:
         if "taxonname" not in val_entry:
             add_unit_inputs[param_name] = None