Tags/annotations editing + dependency modernization + CI modernization

.github/workflows/publish.yml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
 
     - name: Set up Python 3.12
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: 3.12
 
@@ -35,7 +35,7 @@ jobs:
         python -m build
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: python-package-distributions
         path: dist/
@@ -56,7 +56,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v8
       with:
         name: python-package-distributions
         path: dist/
@@ -83,7 +83,7 @@ jobs:
 
     steps:
     - name: Download artifact
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v8
       with:
         name: python-package-distributions
         path: dist/
@@ -107,13 +107,13 @@ jobs:
 
     steps:
     - name: Download artifact
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v8
       with:
         name: python-package-distributions
         path: dist/
 
     - name: Sign with Sigstore
-      uses: sigstore/gh-action-sigstore-python@v2.1.1
+      uses: sigstore/gh-action-sigstore-python@v3
       with:
         inputs: >-
           ./dist/*.tar.gz

.github/workflows/test.yml

@@ -14,22 +14,22 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
-        mongodb-version: ['4.2', '4.4', '5.0', '6.0']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+        mongodb-version: ['5.0', '6.0', '7.0', '8.0']
         dservercore-version: [ main ]
         dserver-search-plugin-mongo-version: [ main ]
 
     steps:
     - name: Git checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Set up python3 ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Start MongoDB
-      uses: supercharge/mongodb-github-action@1.11.0
+      uses: supercharge/mongodb-github-action@1.12.1
       with:
         mongodb-version: ${{ matrix.mongodb-version }}
 

.gitignore

@@ -0,0 +1,24 @@
+.idea
+app.db
+TODO.rst
+.coverage
+
+*.swp
+*.pyc
+*.egg-info
+*.pdf
+
+test_rsa*
+migrations/*
+data/*
+env*
+.cache/*
+.pytest_cache/*
+.tox/*
+build/*
+dist/*
+venv/*
+old-provision/*
+jwt-spike/*
+
+dserver_retrieve_plugin_mongo/version.py

CHANGELOG.rst

@@ -4,6 +4,23 @@ CHANGELOG
 This project uses `semantic versioning <http://semver.org/>`_.
 This change log uses principles from `keep a changelog <http://keepachangelog.com/>`_.
 
+[Unreleased]
+------------
+
+Changed
+^^^^^^^
+
+- Dropped support for Python 3.7–3.9; minimum is now Python 3.10
+- Updated CI Python matrix to 3.10–3.13; updated MongoDB matrix to 5.0–8.0
+
+Fixed
+^^^^^
+
+- Test fixtures: hardcoded MongoDB URI replaced by ``TEST_MONGO_URI`` environment variable (default: ``mongodb://localhost:27017/``)
+- Test fixtures: added ``client.close()`` after ``drop_database()`` in teardown to prevent connection pool exhaustion
+- Test fixtures: removed stale ``FLASK_ENV`` key from app config (removed in Flask 3.0)
+- Config route test: URI assertion now reads from ``TEST_MONGO_URI`` environment variable
+
 [0.4.2]
 -------
 

CLAUDE.md

@@ -0,0 +1,122 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+A retrieve plugin for **dserver** (the dtool lookup server, package `dservercore`). It implements
+MongoDB-backed dataset registration and retrieval of the per-dataset documents (README, manifest,
+annotations, tags). The plugin is discovered by dserver at runtime through the `dservercore.retrieve`
+entry point declared in `pyproject.toml`:
+
+```
+[project.entry-points."dservercore.retrieve"]
+MongoRetrieve = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve"
+```
+
+This package is a *companion* to `dserver-search-plugin-mongo`: retrieve and search are separate
+plugins that, in the common deployment, point at the *same* MongoDB collection (the test fixtures
+wire `RETRIEVE_MONGO_*` and `SEARCH_MONGO_*` to one temp database). The search plugin answers queries
+and returns trimmed dataset records; this plugin fetches the heavy per-dataset content by URI.
+
+## Architecture
+
+The entire plugin is one class, `MongoRetrieve(RetrieveABC)` in
+`dserver_retrieve_plugin_mongo/utils_retrieve.py`, implementing the abstract interface from
+`dservercore.RetrieveABC`:
+
+- `init_app(app)` — pulls `RETRIEVE_MONGO_URI`/`_DB`/`_COLLECTION` from Flask `app.config` and opens
+  the `MongoClient`. Unlike the search plugin, it does *not* create any index.
+- `register_dataset(dataset_info)` — upserts a dataset record keyed on `(uuid, uri)`. Converts
+  `frozen_at`/`created_at` to datetimes via `dservercore.date_utils`. Wraps
+  `pymongo.errors.DocumentTooLarge` into `dservercore.ValidationError`. Also stores a parsed copy of
+  the README under `readme_parsed` (see below).
+- Getters `get_readme(uri)` / `get_manifest(uri)` / `get_annotations(uri)` / `get_tags(uri)` — fetch
+  the record by `uri` and return the corresponding field; raise `dservercore.UnknownURIError` when no
+  record matches.
+- Mutators `set_annotations(uri, …)` / `set_tags(uri, …)` / `set_readme(uri, …)` — `$set` the
+  respective field on the matching record, raising `UnknownURIError` when no record matches.
+  `set_readme` also re-parses and updates `readme_parsed`.
+- `get_config` / `get_config_secrets_to_obfuscate` — expose `config.Config` and the secret list to
+  dserver's `/config` route.
+
+Note `register_dataset` and the `(uuid, uri)` upsert logic are duplicated near-verbatim from the
+search plugin — the two plugins register the same documents into the shared collection.
+
+### README parsing (`readme_parsed`)
+
+The README is stored verbatim as a string under `readme`. `_parse_readme(readme)` additionally
+YAML-parses it (returning the dict, or `None` on failure / non-dict) and the result is stored under
+`readme_parsed`. This enables structured queries over README content by *co-installed* plugins —
+e.g. the dependency-graph plugin's `readme_parsed.derived_from.uuid` dependency key. `readme_parsed`
+is written by `register_dataset` and refreshed by `set_readme`. Requires `PyYAML`.
+
+### Configuration
+
+`config.py` defines `Config` (env-var-backed defaults) and `CONFIG_SECRETS_TO_OBFUSCATE`. Note the
+default `RETRIEVE_MONGO_DB` in `Config` is `"dtool_info"`, while the README example uses `"dserver"`.
+
+## Commands
+
+```bash
+# Install for development (with test deps)
+pip install .[test]
+
+# The test extras also need the server core + search plugin:
+pip install dservercore dserver-search-plugin-mongo
+
+# Run the full test suite (pytest config + coverage live in pyproject.toml)
+pytest
+
+# Run a single test file / single test
+pytest tests/test_utils_retrieve_standalone.py
+pytest tests/test_utils_retrieve_standalone.py::test_functional
+
+# Lint (matches CI)
+flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+flake8 .   # full run
+```
+
+### A running MongoDB is required for most tests
+
+Tests connect to `mongodb://localhost:27017` by default, overridable via the `TEST_MONGO_URI` env var
+(used in `tests/test_utils_retrieve_standalone.py`, `tests/conftest.py` and
+`tests/test_config_route.py`, e.g. for an authenticated MongoDB). They create a randomly-named temp
+database per test and drop it (and close the client) on teardown.
+
+The test app config also sets bare `MONGO_URI`/`MONGO_DB`/`MONGO_COLLECTION` keys (in addition to the
+`RETRIEVE_MONGO_*` / `SEARCH_MONGO_*` ones) so co-installed plugins like the dependency-graph plugin
+can initialise against the same temp database.
+
+## Testing structure
+
+- `tests/test_utils_retrieve_standalone.py` — exercises `MongoRetrieve` directly via a `_MockApp`
+  holding a `config` dict (no Flask). Builds real dtool datasets with `DataSetCreator`, registers
+  them, and asserts the getters/mutators. This is the file to extend when changing retrieval logic.
+- `tests/test_config_route.py` and `tests/conftest.py` — full Flask-app integration via
+  `dservercore.create_app`. `tmp_app_with_users` builds an in-memory SQLite app with JWT users and
+  permissions, wiring both retrieve and search plugins to the same temp Mongo db. Use this for
+  route-level behavior (e.g. `/config/info`, `/config/versions`).
+- `tests/utils.py` — `compare_nested` does partial/marked dict comparison; `make_marker` builds the
+  comparison mask.
+
+## Versioning & packaging
+
+The build backend is **flit** (`flit_scm:buildapi`, configured in `pyproject.toml`); there is no
+`setup.cfg`/`setup.py`. pytest and coverage config also live in `pyproject.toml` under
+`[tool.pytest.ini_options]`.
+
+Version is managed by `setuptools_scm` (via `flit_scm`) from git tags (`guess-next-dev`,
+`no-local-version`), written to `dserver_retrieve_plugin_mongo/version.py` (generated, not committed).
+`__init__.py` resolves the version at runtime first via `importlib.metadata`, falling back to the
+generated `version.py`.
+
+Releases are tag-driven: pushing a tag triggers `.github/workflows/publish.yml` (trusted publishing
+to PyPI + GitHub release + Zenodo). Update `CHANGELOG.rst` (keep-a-changelog format, semver) when
+preparing a release.
+
+## CI
+
+`.github/workflows/test.yml` runs a matrix of Python 3.10–3.13 × MongoDB 5.0/6.0/7.0/8.0, installing
+`dservercore` and `dserver-search-plugin-mongo` from their `main` branches. Keep the plugin
+compatible with that whole range; `pyproject.toml` declares `requires-python = ">=3.10"` to match.

dserver_retrieve_plugin_mongo/utils_retrieve.py

@@ -1,5 +1,7 @@
 """Mongo retrieve plugin module."""
 
+import yaml
+
 import pymongo.errors
 
 from pymongo import MongoClient
@@ -15,6 +17,26 @@
     Config, CONFIG_SECRETS_TO_OBFUSCATE)
 
 
+def _parse_readme(readme):
+    """Return the README parsed into a dict, or None.
+
+    The README is stored verbatim as a string under 'readme'. The parsed
+    representation stored under 'readme_parsed' enables structured queries
+    over README content, e.g. the dependency graph plugin's
+    'readme_parsed.derived_from.uuid' dependency key.
+    """
+    if isinstance(readme, dict):
+        return readme
+    if isinstance(readme, str):
+        try:
+            parsed = yaml.safe_load(readme)
+        except yaml.YAMLError:
+            return None
+        if isinstance(parsed, dict):
+            return parsed
+    return None
+
+
 def _register_dataset_descriptive_metadata(collection, dataset_info):
     """Register dataset info in the collection.
 
@@ -29,6 +51,10 @@ def _register_dataset_descriptive_metadata(collection, dataset_info):
     # get mangled by the datetime replacements.
     dataset_info = dataset_info.copy()
 
+    # Store a parsed representation of the README alongside the verbatim
+    # string to enable structured queries over README content.
+    dataset_info["readme_parsed"] = _parse_readme(dataset_info.get("readme"))
+
     frozen_at = extract_frozen_at_as_datetime(dataset_info)
     created_at = extract_created_at_as_datetime(dataset_info)
 
@@ -103,13 +129,44 @@ def get_annotations(self, uri):
             raise (UnknownURIError())
         return item["annotations"]
 
+    def set_annotations(self, uri, annotations):
+        """Set a dataset's annotations (replaces existing annotations)."""
+        result = self.collection.update_one(
+            {"uri": uri},
+            {"$set": {"annotations": annotations}}
+        )
+        if result.matched_count == 0:
+            raise (UnknownURIError())
+        return annotations
+
     def get_tags(self, uri):
         """Return a dataset's tags."""
         item = self.collection.find_one({"uri": uri})
         if item is None:
             raise (UnknownURIError())
         return item["tags"]
 
+    def set_tags(self, uri, tags):
+        """Set a dataset's tags (replaces existing tags)."""
+        result = self.collection.update_one(
+            {"uri": uri},
+            {"$set": {"tags": tags}}
+        )
+        if result.matched_count == 0:
+            raise (UnknownURIError())
+        return tags
+
+    def set_readme(self, uri, readme):
+        """Set a dataset's readme content."""
+        result = self.collection.update_one(
+            {"uri": uri},
+            {"$set": {"readme": readme,
+                      "readme_parsed": _parse_readme(readme)}}
+        )
+        if result.matched_count == 0:
+            raise (UnknownURIError())
+        return readme
+
     def get_config(self):
         """Return initial Config object, available app-instance independent."""
         return Config