diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1076df7..c93779f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,12 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.12 @@ -35,7 +35,7 @@ jobs: python -m build - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: python-package-distributions path: dist/ @@ -56,7 +56,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: python-package-distributions path: dist/ @@ -83,7 +83,7 @@ jobs: steps: - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: python-package-distributions path: dist/ @@ -107,13 +107,13 @@ jobs: steps: - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: python-package-distributions path: dist/ - name: Sign with Sigstore - uses: sigstore/gh-action-sigstore-python@v2.1.1 + uses: sigstore/gh-action-sigstore-python@v3 with: inputs: >- ./dist/*.tar.gz diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed9e8b0..fe08aa1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,22 +14,22 @@ jobs: strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - mongodb-version: ['4.2', '4.4', '5.0', '6.0'] + python-version: ['3.10', '3.11', '3.12', '3.13'] + mongodb-version: ['5.0', '6.0', '7.0', '8.0'] dservercore-version: [ main ] dserver-search-plugin-mongo-version: [ main ] steps: - name: Git checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up python3 ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.11.0 + uses: supercharge/mongodb-github-action@1.12.1 with: mongodb-version: ${{ matrix.mongodb-version }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9ad2b9a --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +.idea +app.db +TODO.rst +.coverage + +*.swp +*.pyc +*.egg-info +*.pdf + +test_rsa* +migrations/* +data/* +env* +.cache/* +.pytest_cache/* +.tox/* +build/* +dist/* +venv/* +old-provision/* +jwt-spike/* + +dserver_retrieve_plugin_mongo/version.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8931b40..e3f9c0d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,23 @@ CHANGELOG This project uses `semantic versioning `_. This change log uses principles from `keep a changelog `_. +[Unreleased] +------------ + +Changed +^^^^^^^ + +- Dropped support for Python 3.7–3.9; minimum is now Python 3.10 +- Updated CI Python matrix to 3.10–3.13; updated MongoDB matrix to 5.0–8.0 + +Fixed +^^^^^ + +- Test fixtures: hardcoded MongoDB URI replaced by ``TEST_MONGO_URI`` environment variable (default: ``mongodb://localhost:27017/``) +- Test fixtures: added ``client.close()`` after ``drop_database()`` in teardown to prevent connection pool exhaustion +- Test fixtures: removed stale ``FLASK_ENV`` key from app config (removed in Flask 3.0) +- Config route test: URI assertion now reads from ``TEST_MONGO_URI`` environment variable + [0.4.2] ------- diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b9c6c99 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,122 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +A retrieve plugin for **dserver** (the dtool lookup server, package `dservercore`). It implements +MongoDB-backed dataset registration and retrieval of the per-dataset documents (README, manifest, +annotations, tags). The plugin is discovered by dserver at runtime through the `dservercore.retrieve` +entry point declared in `pyproject.toml`: + +``` +[project.entry-points."dservercore.retrieve"] +MongoRetrieve = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve" +``` + +This package is a *companion* to `dserver-search-plugin-mongo`: retrieve and search are separate +plugins that, in the common deployment, point at the *same* MongoDB collection (the test fixtures +wire `RETRIEVE_MONGO_*` and `SEARCH_MONGO_*` to one temp database). The search plugin answers queries +and returns trimmed dataset records; this plugin fetches the heavy per-dataset content by URI. + +## Architecture + +The entire plugin is one class, `MongoRetrieve(RetrieveABC)` in +`dserver_retrieve_plugin_mongo/utils_retrieve.py`, implementing the abstract interface from +`dservercore.RetrieveABC`: + +- `init_app(app)` — pulls `RETRIEVE_MONGO_URI`/`_DB`/`_COLLECTION` from Flask `app.config` and opens + the `MongoClient`. Unlike the search plugin, it does *not* create any index. +- `register_dataset(dataset_info)` — upserts a dataset record keyed on `(uuid, uri)`. Converts + `frozen_at`/`created_at` to datetimes via `dservercore.date_utils`. Wraps + `pymongo.errors.DocumentTooLarge` into `dservercore.ValidationError`. Also stores a parsed copy of + the README under `readme_parsed` (see below). +- Getters `get_readme(uri)` / `get_manifest(uri)` / `get_annotations(uri)` / `get_tags(uri)` — fetch + the record by `uri` and return the corresponding field; raise `dservercore.UnknownURIError` when no + record matches. +- Mutators `set_annotations(uri, …)` / `set_tags(uri, …)` / `set_readme(uri, …)` — `$set` the + respective field on the matching record, raising `UnknownURIError` when no record matches. + `set_readme` also re-parses and updates `readme_parsed`. +- `get_config` / `get_config_secrets_to_obfuscate` — expose `config.Config` and the secret list to + dserver's `/config` route. + +Note `register_dataset` and the `(uuid, uri)` upsert logic are duplicated near-verbatim from the +search plugin — the two plugins register the same documents into the shared collection. + +### README parsing (`readme_parsed`) + +The README is stored verbatim as a string under `readme`. `_parse_readme(readme)` additionally +YAML-parses it (returning the dict, or `None` on failure / non-dict) and the result is stored under +`readme_parsed`. This enables structured queries over README content by *co-installed* plugins — +e.g. the dependency-graph plugin's `readme_parsed.derived_from.uuid` dependency key. `readme_parsed` +is written by `register_dataset` and refreshed by `set_readme`. Requires `PyYAML`. + +### Configuration + +`config.py` defines `Config` (env-var-backed defaults) and `CONFIG_SECRETS_TO_OBFUSCATE`. Note the +default `RETRIEVE_MONGO_DB` in `Config` is `"dtool_info"`, while the README example uses `"dserver"`. + +## Commands + +```bash +# Install for development (with test deps) +pip install .[test] + +# The test extras also need the server core + search plugin: +pip install dservercore dserver-search-plugin-mongo + +# Run the full test suite (pytest config + coverage live in pyproject.toml) +pytest + +# Run a single test file / single test +pytest tests/test_utils_retrieve_standalone.py +pytest tests/test_utils_retrieve_standalone.py::test_functional + +# Lint (matches CI) +flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics +flake8 . # full run +``` + +### A running MongoDB is required for most tests + +Tests connect to `mongodb://localhost:27017` by default, overridable via the `TEST_MONGO_URI` env var +(used in `tests/test_utils_retrieve_standalone.py`, `tests/conftest.py` and +`tests/test_config_route.py`, e.g. for an authenticated MongoDB). They create a randomly-named temp +database per test and drop it (and close the client) on teardown. + +The test app config also sets bare `MONGO_URI`/`MONGO_DB`/`MONGO_COLLECTION` keys (in addition to the +`RETRIEVE_MONGO_*` / `SEARCH_MONGO_*` ones) so co-installed plugins like the dependency-graph plugin +can initialise against the same temp database. + +## Testing structure + +- `tests/test_utils_retrieve_standalone.py` — exercises `MongoRetrieve` directly via a `_MockApp` + holding a `config` dict (no Flask). Builds real dtool datasets with `DataSetCreator`, registers + them, and asserts the getters/mutators. This is the file to extend when changing retrieval logic. +- `tests/test_config_route.py` and `tests/conftest.py` — full Flask-app integration via + `dservercore.create_app`. `tmp_app_with_users` builds an in-memory SQLite app with JWT users and + permissions, wiring both retrieve and search plugins to the same temp Mongo db. Use this for + route-level behavior (e.g. `/config/info`, `/config/versions`). +- `tests/utils.py` — `compare_nested` does partial/marked dict comparison; `make_marker` builds the + comparison mask. + +## Versioning & packaging + +The build backend is **flit** (`flit_scm:buildapi`, configured in `pyproject.toml`); there is no +`setup.cfg`/`setup.py`. pytest and coverage config also live in `pyproject.toml` under +`[tool.pytest.ini_options]`. + +Version is managed by `setuptools_scm` (via `flit_scm`) from git tags (`guess-next-dev`, +`no-local-version`), written to `dserver_retrieve_plugin_mongo/version.py` (generated, not committed). +`__init__.py` resolves the version at runtime first via `importlib.metadata`, falling back to the +generated `version.py`. + +Releases are tag-driven: pushing a tag triggers `.github/workflows/publish.yml` (trusted publishing +to PyPI + GitHub release + Zenodo). Update `CHANGELOG.rst` (keep-a-changelog format, semver) when +preparing a release. + +## CI + +`.github/workflows/test.yml` runs a matrix of Python 3.10–3.13 × MongoDB 5.0/6.0/7.0/8.0, installing +`dservercore` and `dserver-search-plugin-mongo` from their `main` branches. Keep the plugin +compatible with that whole range; `pyproject.toml` declares `requires-python = ">=3.10"` to match. diff --git a/dserver_retrieve_plugin_mongo/utils_retrieve.py b/dserver_retrieve_plugin_mongo/utils_retrieve.py index 3b94c4c..7456dfb 100644 --- a/dserver_retrieve_plugin_mongo/utils_retrieve.py +++ b/dserver_retrieve_plugin_mongo/utils_retrieve.py @@ -1,5 +1,7 @@ """Mongo retrieve plugin module.""" +import yaml + import pymongo.errors from pymongo import MongoClient @@ -15,6 +17,26 @@ Config, CONFIG_SECRETS_TO_OBFUSCATE) +def _parse_readme(readme): + """Return the README parsed into a dict, or None. + + The README is stored verbatim as a string under 'readme'. The parsed + representation stored under 'readme_parsed' enables structured queries + over README content, e.g. the dependency graph plugin's + 'readme_parsed.derived_from.uuid' dependency key. + """ + if isinstance(readme, dict): + return readme + if isinstance(readme, str): + try: + parsed = yaml.safe_load(readme) + except yaml.YAMLError: + return None + if isinstance(parsed, dict): + return parsed + return None + + def _register_dataset_descriptive_metadata(collection, dataset_info): """Register dataset info in the collection. @@ -29,6 +51,10 @@ def _register_dataset_descriptive_metadata(collection, dataset_info): # get mangled by the datetime replacements. dataset_info = dataset_info.copy() + # Store a parsed representation of the README alongside the verbatim + # string to enable structured queries over README content. + dataset_info["readme_parsed"] = _parse_readme(dataset_info.get("readme")) + frozen_at = extract_frozen_at_as_datetime(dataset_info) created_at = extract_created_at_as_datetime(dataset_info) @@ -103,6 +129,16 @@ def get_annotations(self, uri): raise (UnknownURIError()) return item["annotations"] + def set_annotations(self, uri, annotations): + """Set a dataset's annotations (replaces existing annotations).""" + result = self.collection.update_one( + {"uri": uri}, + {"$set": {"annotations": annotations}} + ) + if result.matched_count == 0: + raise (UnknownURIError()) + return annotations + def get_tags(self, uri): """Return a dataset's tags.""" item = self.collection.find_one({"uri": uri}) @@ -110,6 +146,27 @@ def get_tags(self, uri): raise (UnknownURIError()) return item["tags"] + def set_tags(self, uri, tags): + """Set a dataset's tags (replaces existing tags).""" + result = self.collection.update_one( + {"uri": uri}, + {"$set": {"tags": tags}} + ) + if result.matched_count == 0: + raise (UnknownURIError()) + return tags + + def set_readme(self, uri, readme): + """Set a dataset's readme content.""" + result = self.collection.update_one( + {"uri": uri}, + {"$set": {"readme": readme, + "readme_parsed": _parse_readme(readme)}} + ) + if result.matched_count == 0: + raise (UnknownURIError()) + return readme + def get_config(self): """Return initial Config object, available app-instance independent.""" return Config diff --git a/pyproject.toml b/pyproject.toml index 5829c47..13b06bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,24 @@ [build-system] -requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"] -build-backend = "setuptools.build_meta" +requires = ["flit_scm"] +build-backend = "flit_scm:buildapi" [project] name = "dserver-retrieve-plugin-mongo" description = "Retrieve plugin for dserver using mongodb" readme = "README.rst" -license = {file = "LICENSE"} +license = {text = "MIT"} authors = [ {name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"}, {name = "Johannes L. Hörmann", email = "johannes.laurin@gmail.com"}, ] dynamic = ["version"] +requires-python = ">=3.10" dependencies = [ - "pymongo", - "dtoolcore>=3.18.0", - "dservercore" - ] + "pymongo", + "PyYAML", + "dtoolcore>=3.18.0", + "dservercore" +] [project.optional-dependencies] test = [ @@ -31,13 +33,17 @@ Documentation = "https://github.com/jic-dtool/dserver-retrieve-plugin-mongo/blob Repository = "https://github.com/jic-dtool/dserver-retrieve-plugin-mongo" Changelog = "https://github.com/jic-dtool/dserver-retrieve-plugin-mongo/blob/main/CHANGELOG.rst" +[project.entry-points."dservercore.retrieve"] +MongoRetrieve = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve" + +[tool.flit.module] +name = "dserver_retrieve_plugin_mongo" + [tool.setuptools_scm] version_scheme = "guess-next-dev" local_scheme = "no-local-version" write_to = "dserver_retrieve_plugin_mongo/version.py" -[tool.setuptools] -packages = ["dserver_retrieve_plugin_mongo"] - -[project.entry-points."dservercore.retrieve"] -"MongoRetrieve" = "dserver_retrieve_plugin_mongo.utils_retrieve:MongoRetrieve" +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=dserver_retrieve_plugin_mongo --cov-report=term-missing" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3f525fd..0000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -exclude=env*,.tox,.git,*.egg,build - -[tool:pytest] -testpaths = tests -addopts = --cov=dserver_retrieve_plugin_mongo -#addopts = -x --pdb - -[cov:run] -source = dserver_retrieve_plugin_mongo diff --git a/tests/conftest.py b/tests/conftest.py index a436a43..f90f763 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Reusable fixtures""" +import os import random import string @@ -7,6 +8,7 @@ JWT_PUBLIC_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8LrEp0Q6l1WPsY32uOPqEjaisQScnzO/XvlhQTzj5w+hFObjiNgIaHRceYh3hZZwsRsHIkCxOY0JgUPeFP9IVXso0VptIjCPRF5yrV/+dF1rtl4eyYj/XOBvSDzbQQwqdjhHffw0TXW0f/yjGGJCYM+tw/9dmj9VilAMNTx1H76uPKUo4M3vLBQLo2tj7z1jlh4Jlw5hKBRcWQWbpWP95p71Db6gSpqReDYbx57BW19APMVketUYsXfXTztM/HWz35J9HDya3ID0Dl+pE22Wo8SZo2+ULKu/4OYVcD8DjF15WwXrcuFDypX132j+LUWOVWxCs5hdMybSDwF3ZhVBH ec2-user@ip-172-31-41-191.eu-west-1.compute.internal" # NOQA +MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/") def random_string( @@ -41,14 +43,18 @@ def tmp_app_with_users(request): "OPENAPI_VERSION": '3.0.2', "CONFIG_SECRETS_TO_OBFUSCATE": [], "SECRET_KEY": "secret", - "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", - "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/", + "RETRIEVE_MONGO_URI": MONGO_URI, "RETRIEVE_MONGO_DB": tmp_mongo_db_name, "RETRIEVE_MONGO_COLLECTION": "datasets", - "SEARCH_MONGO_URI": "mongodb://localhost:27017/", + "SEARCH_MONGO_URI": MONGO_URI, "SEARCH_MONGO_DB": tmp_mongo_db_name, "SEARCH_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db_name, + "MONGO_COLLECTION": "datasets", "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, @@ -84,7 +90,9 @@ def tmp_app_with_users(request): @request.addfinalizer def teardown(): current_app.retrieve.client.drop_database(tmp_mongo_db_name) + current_app.retrieve.client.close() current_app.search.client.drop_database(tmp_mongo_db_name) + current_app.search.client.close() sql_db.session.remove() - return app.test_client() \ No newline at end of file + return app.test_client() diff --git a/tests/test_config_route.py b/tests/test_config_route.py index e81125a..9f8506e 100644 --- a/tests/test_config_route.py +++ b/tests/test_config_route.py @@ -1,6 +1,7 @@ """Test the /config blueprint route.""" import json +import os import dserver_retrieve_plugin_mongo from .utils import compare_nested @@ -17,7 +18,7 @@ def test_config_info_route(tmp_app_with_users, snowwhite_token): # NOQA expected_content = { 'retrieve_mongo_collection': 'datasets', - 'retrieve_mongo_uri': 'mongodb://localhost:27017/', + 'retrieve_mongo_uri': os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), } response = json.loads(r.data.decode("utf-8")) diff --git a/tests/test_utils_retrieve_standalone.py b/tests/test_utils_retrieve_standalone.py index 410fa34..df54f3b 100644 --- a/tests/test_utils_retrieve_standalone.py +++ b/tests/test_utils_retrieve_standalone.py @@ -11,6 +11,8 @@ import pytest +import os + from pymongo import MongoClient from dtoolcore import DataSetCreator, DataSet @@ -20,8 +22,7 @@ # This tested in this module. from dserver_retrieve_plugin_mongo.utils_retrieve import MongoRetrieve - -MONGO_URI = "mongodb://localhost:27017" +MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/") def random_string( @@ -40,6 +41,7 @@ def tmp_mongo_db(request): @request.addfinalizer def teardown(): client.drop_database(tmp_mongo_db_name) + client.close() return tmp_mongo_db_name @@ -111,7 +113,12 @@ def test_functional(tmp_mongo_db): # NOQA app.config = { "RETRIEVE_MONGO_URI": MONGO_URI, "RETRIEVE_MONGO_DB": tmp_mongo_db, - "RETRIEVE_MONGO_COLLECTION": "datasets" + "RETRIEVE_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db, + "MONGO_COLLECTION": "datasets", } mongo_retreive.init_app(app) @@ -165,7 +172,12 @@ def test_register_raises_when_metadata_too_large(tmp_mongo_db): # NOQA app.config = { "RETRIEVE_MONGO_URI": MONGO_URI, "RETRIEVE_MONGO_DB": tmp_mongo_db, - "RETRIEVE_MONGO_COLLECTION": "datasets" + "RETRIEVE_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db, + "MONGO_COLLECTION": "datasets", } mongo_retreive.init_app(app)