diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 1076df7..c93779f 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -14,12 +14,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Set up Python 3.12
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: 3.12
@@ -35,7 +35,7 @@ jobs:
python -m build
- name: Upload artifact
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: python-package-distributions
path: dist/
@@ -56,7 +56,7 @@ jobs:
steps:
- name: Download all the dists
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: python-package-distributions
path: dist/
@@ -83,7 +83,7 @@ jobs:
steps:
- name: Download artifact
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: python-package-distributions
path: dist/
@@ -107,13 +107,13 @@ jobs:
steps:
- name: Download artifact
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: python-package-distributions
path: dist/
- name: Sign with Sigstore
- uses: sigstore/gh-action-sigstore-python@v2.1.1
+ uses: sigstore/gh-action-sigstore-python@v3
with:
inputs: >-
./dist/*.tar.gz
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ed9e8b0..fe08aa1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,22 +14,22 @@ jobs:
strategy:
matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
- mongodb-version: ['4.2', '4.4', '5.0', '6.0']
+ python-version: ['3.10', '3.11', '3.12', '3.13']
+ mongodb-version: ['5.0', '6.0', '7.0', '8.0']
dservercore-version: [ main ]
dserver-search-plugin-mongo-version: [ main ]
steps:
- name: Git checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: Set up python3 ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Start MongoDB
- uses: supercharge/mongodb-github-action@1.11.0
+ uses: supercharge/mongodb-github-action@1.12.1
with:
mongodb-version: ${{ matrix.mongodb-version }}
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9ad2b9a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,24 @@
+.idea
+app.db
+TODO.rst
+.coverage
+
+*.swp
+*.pyc
+*.egg-info
+*.pdf
+
+test_rsa*
+migrations/*
+data/*
+env*
+.cache/*
+.pytest_cache/*
+.tox/*
+build/*
+dist/*
+venv/*
+old-provision/*
+jwt-spike/*
+
+dserver_retrieve_plugin_mongo/version.py
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 8931b40..e3f9c0d 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -4,6 +4,23 @@ CHANGELOG
This project uses `semantic versioning `_.
This change log uses principles from `keep a changelog `_.
+[Unreleased]
+------------
+
+Changed
+^^^^^^^
+
+- Dropped support for Python 3.7–3.9; minimum is now Python 3.10
+- Updated CI Python matrix to 3.10–3.13; updated MongoDB matrix to 5.0–8.0
+
+Fixed
+^^^^^
+
+- Test fixtures: hardcoded MongoDB URI replaced by ``TEST_MONGO_URI`` environment variable (default: ``mongodb://localhost:27017/``)
+- Test fixtures: added ``client.close()`` after ``drop_database()`` in teardown to prevent connection pool exhaustion
+- Test fixtures: removed stale ``FLASK_ENV`` key from app config (removed in Flask 3.0)
+- Config route test: URI assertion now reads from ``TEST_MONGO_URI`` environment variable
+
[0.4.2]
-------
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..b9c6c99
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,122 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+A retrieve plugin for **dserver** (the dtool lookup server, package `dservercore`). It implements
+MongoDB-backed dataset registration and retrieval of the per-dataset documents (README, manifest,
+annotations, tags). The plugin is discovered by dserver at runtime through the `dservercore.retrieve`
+entry point declared in `pyproject.toml`:
+
+```
+[project.entry-points."dservercore.retrieve"]
+MongoRetrieve = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve"
+```
+
+This package is a *companion* to `dserver-search-plugin-mongo`: retrieve and search are separate
+plugins that, in the common deployment, point at the *same* MongoDB collection (the test fixtures
+wire `RETRIEVE_MONGO_*` and `SEARCH_MONGO_*` to one temp database). The search plugin answers queries
+and returns trimmed dataset records; this plugin fetches the heavy per-dataset content by URI.
+
+## Architecture
+
+The entire plugin is one class, `MongoRetrieve(RetrieveABC)` in
+`dserver_retrieve_plugin_mongo/utils_retrieve.py`, implementing the abstract interface from
+`dservercore.RetrieveABC`:
+
+- `init_app(app)` — pulls `RETRIEVE_MONGO_URI`/`_DB`/`_COLLECTION` from Flask `app.config` and opens
+ the `MongoClient`. Unlike the search plugin, it does *not* create any index.
+- `register_dataset(dataset_info)` — upserts a dataset record keyed on `(uuid, uri)`. Converts
+ `frozen_at`/`created_at` to datetimes via `dservercore.date_utils`. Wraps
+ `pymongo.errors.DocumentTooLarge` into `dservercore.ValidationError`. Also stores a parsed copy of
+ the README under `readme_parsed` (see below).
+- Getters `get_readme(uri)` / `get_manifest(uri)` / `get_annotations(uri)` / `get_tags(uri)` — fetch
+ the record by `uri` and return the corresponding field; raise `dservercore.UnknownURIError` when no
+ record matches.
+- Mutators `set_annotations(uri, …)` / `set_tags(uri, …)` / `set_readme(uri, …)` — `$set` the
+ respective field on the matching record, raising `UnknownURIError` when no record matches.
+ `set_readme` also re-parses and updates `readme_parsed`.
+- `get_config` / `get_config_secrets_to_obfuscate` — expose `config.Config` and the secret list to
+ dserver's `/config` route.
+
+Note `register_dataset` and the `(uuid, uri)` upsert logic are duplicated near-verbatim from the
+search plugin — the two plugins register the same documents into the shared collection.
+
+### README parsing (`readme_parsed`)
+
+The README is stored verbatim as a string under `readme`. `_parse_readme(readme)` additionally
+YAML-parses it (returning the dict, or `None` on failure / non-dict) and the result is stored under
+`readme_parsed`. This enables structured queries over README content by *co-installed* plugins —
+e.g. the dependency-graph plugin's `readme_parsed.derived_from.uuid` dependency key. `readme_parsed`
+is written by `register_dataset` and refreshed by `set_readme`. Requires `PyYAML`.
+
+### Configuration
+
+`config.py` defines `Config` (env-var-backed defaults) and `CONFIG_SECRETS_TO_OBFUSCATE`. Note the
+default `RETRIEVE_MONGO_DB` in `Config` is `"dtool_info"`, while the README example uses `"dserver"`.
+
+## Commands
+
+```bash
+# Install for development (with test deps)
+pip install .[test]
+
+# The test extras also need the server core + search plugin:
+pip install dservercore dserver-search-plugin-mongo
+
+# Run the full test suite (pytest config + coverage live in pyproject.toml)
+pytest
+
+# Run a single test file / single test
+pytest tests/test_utils_retrieve_standalone.py
+pytest tests/test_utils_retrieve_standalone.py::test_functional
+
+# Lint (matches CI)
+flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+flake8 . # full run
+```
+
+### A running MongoDB is required for most tests
+
+Tests connect to `mongodb://localhost:27017` by default, overridable via the `TEST_MONGO_URI` env var
+(used in `tests/test_utils_retrieve_standalone.py`, `tests/conftest.py` and
+`tests/test_config_route.py`, e.g. for an authenticated MongoDB). They create a randomly-named temp
+database per test and drop it (and close the client) on teardown.
+
+The test app config also sets bare `MONGO_URI`/`MONGO_DB`/`MONGO_COLLECTION` keys (in addition to the
+`RETRIEVE_MONGO_*` / `SEARCH_MONGO_*` ones) so co-installed plugins like the dependency-graph plugin
+can initialise against the same temp database.
+
+## Testing structure
+
+- `tests/test_utils_retrieve_standalone.py` — exercises `MongoRetrieve` directly via a `_MockApp`
+ holding a `config` dict (no Flask). Builds real dtool datasets with `DataSetCreator`, registers
+ them, and asserts the getters/mutators. This is the file to extend when changing retrieval logic.
+- `tests/test_config_route.py` and `tests/conftest.py` — full Flask-app integration via
+ `dservercore.create_app`. `tmp_app_with_users` builds an in-memory SQLite app with JWT users and
+ permissions, wiring both retrieve and search plugins to the same temp Mongo db. Use this for
+ route-level behavior (e.g. `/config/info`, `/config/versions`).
+- `tests/utils.py` — `compare_nested` does partial/marked dict comparison; `make_marker` builds the
+ comparison mask.
+
+## Versioning & packaging
+
+The build backend is **flit** (`flit_scm:buildapi`, configured in `pyproject.toml`); there is no
+`setup.cfg`/`setup.py`. pytest and coverage config also live in `pyproject.toml` under
+`[tool.pytest.ini_options]`.
+
+Version is managed by `setuptools_scm` (via `flit_scm`) from git tags (`guess-next-dev`,
+`no-local-version`), written to `dserver_retrieve_plugin_mongo/version.py` (generated, not committed).
+`__init__.py` resolves the version at runtime first via `importlib.metadata`, falling back to the
+generated `version.py`.
+
+Releases are tag-driven: pushing a tag triggers `.github/workflows/publish.yml` (trusted publishing
+to PyPI + GitHub release + Zenodo). Update `CHANGELOG.rst` (keep-a-changelog format, semver) when
+preparing a release.
+
+## CI
+
+`.github/workflows/test.yml` runs a matrix of Python 3.10–3.13 × MongoDB 5.0/6.0/7.0/8.0, installing
+`dservercore` and `dserver-search-plugin-mongo` from their `main` branches. Keep the plugin
+compatible with that whole range; `pyproject.toml` declares `requires-python = ">=3.10"` to match.
diff --git a/dserver_retrieve_plugin_mongo/utils_retrieve.py b/dserver_retrieve_plugin_mongo/utils_retrieve.py
index 3b94c4c..7456dfb 100644
--- a/dserver_retrieve_plugin_mongo/utils_retrieve.py
+++ b/dserver_retrieve_plugin_mongo/utils_retrieve.py
@@ -1,5 +1,7 @@
"""Mongo retrieve plugin module."""
+import yaml
+
import pymongo.errors
from pymongo import MongoClient
@@ -15,6 +17,26 @@
Config, CONFIG_SECRETS_TO_OBFUSCATE)
+def _parse_readme(readme):
+ """Return the README parsed into a dict, or None.
+
+ The README is stored verbatim as a string under 'readme'. The parsed
+ representation stored under 'readme_parsed' enables structured queries
+ over README content, e.g. the dependency graph plugin's
+ 'readme_parsed.derived_from.uuid' dependency key.
+ """
+ if isinstance(readme, dict):
+ return readme
+ if isinstance(readme, str):
+ try:
+ parsed = yaml.safe_load(readme)
+ except yaml.YAMLError:
+ return None
+ if isinstance(parsed, dict):
+ return parsed
+ return None
+
+
def _register_dataset_descriptive_metadata(collection, dataset_info):
"""Register dataset info in the collection.
@@ -29,6 +51,10 @@ def _register_dataset_descriptive_metadata(collection, dataset_info):
# get mangled by the datetime replacements.
dataset_info = dataset_info.copy()
+ # Store a parsed representation of the README alongside the verbatim
+ # string to enable structured queries over README content.
+ dataset_info["readme_parsed"] = _parse_readme(dataset_info.get("readme"))
+
frozen_at = extract_frozen_at_as_datetime(dataset_info)
created_at = extract_created_at_as_datetime(dataset_info)
@@ -103,6 +129,16 @@ def get_annotations(self, uri):
raise (UnknownURIError())
return item["annotations"]
+ def set_annotations(self, uri, annotations):
+ """Set a dataset's annotations (replaces existing annotations)."""
+ result = self.collection.update_one(
+ {"uri": uri},
+ {"$set": {"annotations": annotations}}
+ )
+ if result.matched_count == 0:
+ raise (UnknownURIError())
+ return annotations
+
def get_tags(self, uri):
"""Return a dataset's tags."""
item = self.collection.find_one({"uri": uri})
@@ -110,6 +146,27 @@ def get_tags(self, uri):
raise (UnknownURIError())
return item["tags"]
+ def set_tags(self, uri, tags):
+ """Set a dataset's tags (replaces existing tags)."""
+ result = self.collection.update_one(
+ {"uri": uri},
+ {"$set": {"tags": tags}}
+ )
+ if result.matched_count == 0:
+ raise (UnknownURIError())
+ return tags
+
+ def set_readme(self, uri, readme):
+ """Set a dataset's readme content."""
+ result = self.collection.update_one(
+ {"uri": uri},
+ {"$set": {"readme": readme,
+ "readme_parsed": _parse_readme(readme)}}
+ )
+ if result.matched_count == 0:
+ raise (UnknownURIError())
+ return readme
+
def get_config(self):
"""Return initial Config object, available app-instance independent."""
return Config
diff --git a/pyproject.toml b/pyproject.toml
index 5829c47..13b06bc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,22 +1,24 @@
[build-system]
-requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"]
-build-backend = "setuptools.build_meta"
+requires = ["flit_scm"]
+build-backend = "flit_scm:buildapi"
[project]
name = "dserver-retrieve-plugin-mongo"
description = "Retrieve plugin for dserver using mongodb"
readme = "README.rst"
-license = {file = "LICENSE"}
+license = {text = "MIT"}
authors = [
{name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"},
{name = "Johannes L. Hörmann", email = "johannes.laurin@gmail.com"},
]
dynamic = ["version"]
+requires-python = ">=3.10"
dependencies = [
- "pymongo",
- "dtoolcore>=3.18.0",
- "dservercore"
- ]
+ "pymongo",
+ "PyYAML",
+ "dtoolcore>=3.18.0",
+ "dservercore"
+]
[project.optional-dependencies]
test = [
@@ -31,13 +33,17 @@ Documentation = "https://github.com/jic-dtool/dserver-retrieve-plugin-mongo/blob
Repository = "https://github.com/jic-dtool/dserver-retrieve-plugin-mongo"
Changelog = "https://github.com/jic-dtool/dserver-retrieve-plugin-mongo/blob/main/CHANGELOG.rst"
+[project.entry-points."dservercore.retrieve"]
+MongoRetrieve = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve"
+
+[tool.flit.module]
+name = "dserver_retrieve_plugin_mongo"
+
[tool.setuptools_scm]
version_scheme = "guess-next-dev"
local_scheme = "no-local-version"
write_to = "dserver_retrieve_plugin_mongo/version.py"
-[tool.setuptools]
-packages = ["dserver_retrieve_plugin_mongo"]
-
-[project.entry-points."dservercore.retrieve"]
-"MongoRetrieve" = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve"
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=dserver_retrieve_plugin_mongo --cov-report=term-missing"
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 3f525fd..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,10 +0,0 @@
-[flake8]
-exclude=env*,.tox,.git,*.egg,build
-
-[tool:pytest]
-testpaths = tests
-addopts = --cov=dserver_retrieve_plugin_mongo
-#addopts = -x --pdb
-
-[cov:run]
-source = dserver_retrieve_plugin_mongo
diff --git a/tests/conftest.py b/tests/conftest.py
index a436a43..f90f763 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,5 +1,6 @@
"""Reusable fixtures"""
+import os
import random
import string
@@ -7,6 +8,7 @@
JWT_PUBLIC_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8LrEp0Q6l1WPsY32uOPqEjaisQScnzO/XvlhQTzj5w+hFObjiNgIaHRceYh3hZZwsRsHIkCxOY0JgUPeFP9IVXso0VptIjCPRF5yrV/+dF1rtl4eyYj/XOBvSDzbQQwqdjhHffw0TXW0f/yjGGJCYM+tw/9dmj9VilAMNTx1H76uPKUo4M3vLBQLo2tj7z1jlh4Jlw5hKBRcWQWbpWP95p71Db6gSpqReDYbx57BW19APMVketUYsXfXTztM/HWz35J9HDya3ID0Dl+pE22Wo8SZo2+ULKu/4OYVcD8DjF15WwXrcuFDypX132j+LUWOVWxCs5hdMybSDwF3ZhVBH ec2-user@ip-172-31-41-191.eu-west-1.compute.internal" # NOQA
+MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/")
def random_string(
@@ -41,14 +43,18 @@ def tmp_app_with_users(request):
"OPENAPI_VERSION": '3.0.2',
"CONFIG_SECRETS_TO_OBFUSCATE": [],
"SECRET_KEY": "secret",
- "FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
- "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/",
+ "RETRIEVE_MONGO_URI": MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db_name,
"RETRIEVE_MONGO_COLLECTION": "datasets",
- "SEARCH_MONGO_URI": "mongodb://localhost:27017/",
+ "SEARCH_MONGO_URI": MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db_name,
"SEARCH_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db_name,
+ "MONGO_COLLECTION": "datasets",
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
@@ -84,7 +90,9 @@ def tmp_app_with_users(request):
@request.addfinalizer
def teardown():
current_app.retrieve.client.drop_database(tmp_mongo_db_name)
+ current_app.retrieve.client.close()
current_app.search.client.drop_database(tmp_mongo_db_name)
+ current_app.search.client.close()
sql_db.session.remove()
- return app.test_client()
\ No newline at end of file
+ return app.test_client()
diff --git a/tests/test_config_route.py b/tests/test_config_route.py
index e81125a..9f8506e 100644
--- a/tests/test_config_route.py
+++ b/tests/test_config_route.py
@@ -1,6 +1,7 @@
"""Test the /config blueprint route."""
import json
+import os
import dserver_retrieve_plugin_mongo
from .utils import compare_nested
@@ -17,7 +18,7 @@ def test_config_info_route(tmp_app_with_users, snowwhite_token): # NOQA
expected_content = {
'retrieve_mongo_collection': 'datasets',
- 'retrieve_mongo_uri': 'mongodb://localhost:27017/',
+ 'retrieve_mongo_uri': os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
}
response = json.loads(r.data.decode("utf-8"))
diff --git a/tests/test_utils_retrieve_standalone.py b/tests/test_utils_retrieve_standalone.py
index 410fa34..df54f3b 100644
--- a/tests/test_utils_retrieve_standalone.py
+++ b/tests/test_utils_retrieve_standalone.py
@@ -11,6 +11,8 @@
import pytest
+import os
+
from pymongo import MongoClient
from dtoolcore import DataSetCreator, DataSet
@@ -20,8 +22,7 @@
# This tested in this module.
from dserver_retrieve_plugin_mongo.utils_retrieve import MongoRetrieve
-
-MONGO_URI = "mongodb://localhost:27017"
+MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/")
def random_string(
@@ -40,6 +41,7 @@ def tmp_mongo_db(request):
@request.addfinalizer
def teardown():
client.drop_database(tmp_mongo_db_name)
+ client.close()
return tmp_mongo_db_name
@@ -111,7 +113,12 @@ def test_functional(tmp_mongo_db): # NOQA
app.config = {
"RETRIEVE_MONGO_URI": MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db,
- "RETRIEVE_MONGO_COLLECTION": "datasets"
+ "RETRIEVE_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db,
+ "MONGO_COLLECTION": "datasets",
}
mongo_retreive.init_app(app)
@@ -165,7 +172,12 @@ def test_register_raises_when_metadata_too_large(tmp_mongo_db): # NOQA
app.config = {
"RETRIEVE_MONGO_URI": MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db,
- "RETRIEVE_MONGO_COLLECTION": "datasets"
+ "RETRIEVE_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db,
+ "MONGO_COLLECTION": "datasets",
}
mongo_retreive.init_app(app)