diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 76f4c46..771c82d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -17,7 +17,7 @@ jobs:
strategy:
matrix:
- python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
+ python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
mongodb-version: ['5.0', '6.0', '7.0', '8.0']
dserver-search-plugin-mongo-version: ['0.4.2']
dserver-retrieve-plugin-mongo-version: ['0.4.2']
@@ -59,4 +59,4 @@ jobs:
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
\ No newline at end of file
+ flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 0f78d62..77ff6bf 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -4,8 +4,8 @@ CHANGELOG
This project uses `semantic versioning `_.
This change log uses principles from `keep a changelog `_.
-[0.23.0] - 2025-12-08
----------------------
+[Unreleased]
+------------
Added
^^^^^
@@ -33,7 +33,20 @@ Added
Changed
^^^^^^^
+- Dropped support for Python 3.8 and 3.9; minimum is now Python 3.10
+- Upgraded to Flask 3.x (removed ``flask<3`` upper bound)
+- Removed unused ``flask-pymongo`` dependency
+- Upgraded to marshmallow 4.x, flask-marshmallow 1.5.0, marshmallow-sqlalchemy 1.5.0
+- Removed deprecated ``JSONIFY_PRETTYPRINT_REGULAR`` config key (removed in Flask 3.0)
+- Removed deprecated ``Meta.ordered = True`` from marshmallow schemas in ``sort.py`` (removed in marshmallow 4.x)
+- Updated CI Python matrix to 3.10–3.13; updated MongoDB matrix to 5.0–8.0
+- Fixed ``myst-parser==4.0.0`` hard pin in docs extra to ``myst-parser>=5.0``
+
+Fixed
+^^^^^
+- Test fixtures: hardcoded MongoDB URI replaced by ``MONGO_URI`` environment variable (default: ``mongodb://localhost:27017/``)
+- Test fixtures: added ``client.close()`` after ``drop_database()`` in teardown to prevent connection pool exhaustion
- Switched build system to flit
- Tags and annotations are now updated both in the database and in storage
(previously only updated in database)
@@ -43,7 +56,6 @@ Removed
- Removed unused legacy code
-
[0.22.0]
------------
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..423840c
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,122 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this project is
+
+`dservercore` is the core Flask application for `dserver`, a web API for registering, looking up, and searching dtool dataset metadata. It provides a pluggable framework where the actual storage and search backends are delegated to separately-installed plugins.
+
+## Commands
+
+**Install for development:**
+```bash
+pip install -e ".[test]"
+```
+
+**Run tests** (requires a running MongoDB on localhost:27017):
+```bash
+pytest -sv
+```
+
+If MongoDB requires authentication, pass credentials via `MONGO_URI`:
+```bash
+MONGO_URI="mongodb://user:password@localhost:27017/" pytest -sv
+```
+
+**Run a single test file:**
+```bash
+pytest -sv tests/test_uri_routes.py
+```
+
+**Run a single test:**
+```bash
+pytest -sv tests/test_uri_routes.py::test_name
+```
+
+**Lint:**
+```bash
+flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+```
+
+**Run the app locally:**
+```bash
+export FLASK_APP=dservercore
+flask run
+```
+
+**DB migrations** (after changing SQL models):
+```bash
+flask db init
+flask db migrate
+flask db upgrade
+```
+
+## Architecture
+
+### Plugin system
+
+`dservercore` does not store or search dataset descriptive content itself — that is fully delegated to two required plugins and any number of optional extension plugins, discovered at runtime via Python entry points:
+
+- **`dservercore.search`** — exactly one search plugin must be installed (e.g. `dserver-search-plugin-mongo`). Must subclass `SearchABC` and implement `search()` and `register_dataset()`.
+- **`dservercore.retrieve`** — exactly one retrieve plugin must be installed (e.g. `dserver-retrieve-plugin-mongo`). Must subclass `RetrieveABC` and implement `get_readme()`, `get_manifest()`, `get_annotations()`, `get_tags()`.
+- **`dservercore.extension`** — zero or more optional extensions, subclassing `ExtensionABC`. Must implement `get_blueprint()` and `register_dataset()`.
+
+All plugin ABCs are defined in `dservercore/__init__.py`. Plugins are loaded in `create_app()` and attached to the Flask `app` object as `app.search`, `app.retrieve`, and `app.custom_extensions`. Routes access them via `current_app.search` / `current_app.retrieve`.
+
+### Dual data stores
+
+Dataset metadata is stored in two places simultaneously:
+
+1. **SQL database** (SQLAlchemy, default SQLite) — stores admin metadata (URI, UUID, name, creator, timestamps, size). Models: `User`, `BaseURI`, `Dataset` in `sql_models.py`. This is the authoritative source for listing datasets and permission checks.
+2. **Plugin backends** — the search and retrieve plugins each maintain their own store (e.g. MongoDB) for full descriptive metadata (readme, manifest, annotations, tags).
+
+`register_dataset()` in `utils.py` writes to both. The SQL side is the source for `list_datasets_by_user()` (no query) while the search plugin is used for `search_datasets_by_user()` (with query).
+
+### Authorization model
+
+- **JWT authentication** (RS256, asymmetric key) via `flask-jwt-extended`. The identity extracted from the token is the username.
+- `utils_auth.py` provides `jwt_required()` and `get_jwt_identity()` wrappers that can be disabled for testing via `DISABLE_JWT_AUTHORISATION=True` config flag (returns `DEFAULT_USER` instead).
+- Permissions are stored in SQL: users have `search_permissions` and `register_permissions` on specific `BaseURI` entries (many-to-many via association tables in `sql_models.py`).
+- Admins (`is_admin=True`) bypass base URI permission checks in admin-only routes.
+
+### Routes
+
+Each resource has its own route module (`*_routes.py`) registering a `flask-smorest` Blueprint. All blueprints use the custom `dservercore.blueprint.Blueprint` (which is `FlaskSmorestBlueprint + SortMixin`) to support the `?sort=+field,-other` query parameter convention. Blueprints are registered in `create_app()`.
+
+Route modules:
+- `uri_routes.py` — `GET/POST/PUT/DELETE /uris/` for dataset search, registration, and deletion
+- `uuid_routes.py` — `GET /uuids/` to look up all URIs for a UUID
+- `base_uri_routes.py` — admin management of base URIs
+- `user_routes.py` — admin management of users and permissions
+- `me_routes.py` — self-service route for the authenticated user
+- `manifest_routes.py`, `readme_routes.py`, `annotations_routes.py`, `tags_routes.py` — retrieve plugin pass-through routes
+
+### URI encoding in URL paths
+
+Dataset URIs contain `://` which cannot appear directly in URL paths. `utils.py` provides `url_suffix_to_uri()` and `uri_to_url_suffix()` to convert between `s3://bucket/uuid` and the URL-safe form `s3/bucket/uuid` used in route paths like `GET /uris/s3/bucket/uuid`.
+
+### Sorting
+
+`sort.py` implements a custom sort feature on top of `flask-smorest`. Sort parameters are passed as `?sort=+field,-other` (comma-separated, `+`/`-` prefix for direction). `SortMixin` and the custom `Blueprint` class inject a `sort_parameters: SortParameters` kwarg into route handlers and add an `X-Sort` response header. `_dataset_order_by_args()` in `utils.py` translates `SortParameters` to SQLAlchemy `order_by` args.
+
+### Testing
+
+Tests require a real MongoDB instance on `localhost:27017` (no mocking). The `conftest.py` creates temporary MongoDB databases with randomised names and tears them down after each test. The main fixtures are:
+- `tmp_app` — bare app, no users
+- `tmp_app_with_users` — app with a pre-registered set of users and permissions
+- `tmp_app_with_data` — app with users, base URIs, and dataset entries registered
+- `tmp_cli_runner` — Flask test CLI runner
+
+Pre-signed JWT tokens for test users (`snow-white`, `grumpy`, `dopey`, `sleepy`, `noone`) are hardcoded in `conftest.py` and validated against a hardcoded RSA public key.
+
+## Key configuration variables
+
+| Variable | Purpose |
+|---|---|
+| `FLASK_APP` | Must be set to `dservercore` |
+| `SQLALCHEMY_DATABASE_URI` | SQL backend (default: SQLite `app.db`) |
+| `JWT_PUBLIC_KEY` / `JWT_PUBLIC_KEY_FILE` | RSA public key for token verification |
+| `JWT_PRIVATE_KEY_FILE` | RSA private key (only needed to issue tokens) |
+| `DISABLE_JWT_AUTHORISATION` | Bypass auth for dev/testing |
+| `DEFAULT_USER` | Username used when JWT auth is disabled |
+| `OPENAPI_URL_PREFIX` | Path prefix for OpenAPI docs (default `/doc`) |
diff --git a/dservercore/config.py b/dservercore/config.py
index 43d4e81..3a9a006 100644
--- a/dservercore/config.py
+++ b/dservercore/config.py
@@ -52,8 +52,6 @@ class Config(object):
# If JWT authorisation disabled, always identify as this user:
DEFAULT_USER = os.environ.get("DEFAULT_USER", "testuser")
- JSONIFY_PRETTYPRINT_REGULAR = True
-
API_TITLE = "dserver API"
API_VERSION = "v1"
diff --git a/dservercore/sort.py b/dservercore/sort.py
index 49b48f4..e940e95 100644
--- a/dservercore/sort.py
+++ b/dservercore/sort.py
@@ -103,7 +103,6 @@ class SortParametersSchema(ma.Schema):
"""Deserializes sort params into SortParameters"""
class Meta:
- ordered = True
unknown = ma.EXCLUDE
sort = ma.fields.List(
@@ -130,9 +129,6 @@ class SortMetadataSchema(ma.Schema):
sort = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.Integer())
- class Meta:
- ordered = True
-
SORT_HEADER = {
"description": "Sort metadata",
diff --git a/pyproject.toml b/pyproject.toml
index 015b50a..9cd5214 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,23 +11,22 @@ authors = [
{name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"}
]
dynamic = ["version"]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
dependencies = [
- "flask<3",
- "pymongo",
- "alembic",
- "flask-sqlalchemy",
- "flask-migrate",
- "flask-pymongo",
- "flask-marshmallow",
- "flask-smorest",
- "marshmallow-sqlalchemy",
- "flask-cors",
- "dtoolcore>=3.18.0",
- "flask-jwt-extended[asymmetric_crypto]>=4.6.0",
- "pyyaml",
- "marshmallow<4.0.0"
-]
+ "setuptools",
+ "flask",
+ "pymongo",
+ "alembic",
+ "flask-sqlalchemy",
+ "flask-migrate",
+ "flask-marshmallow",
+ "flask-smorest",
+ "marshmallow-sqlalchemy",
+ "flask-cors",
+ "dtoolcore>=3.18.0",
+ "flask-jwt-extended[asymmetric_crypto]>=4.7.0",
+ "pyyaml"
+ ]
[project.optional-dependencies]
test = [
@@ -40,7 +39,7 @@ docs = [
"sphinx",
"sphinx_rtd_theme",
"sphinxcontrib-spelling",
- "myst-parser==4.0.0"
+ "myst-parser>=5.0"
]
[project.urls]
diff --git a/tests/conftest.py b/tests/conftest.py
index b649446..681f00a 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -16,6 +16,8 @@
JWT_PUBLIC_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8LrEp0Q6l1WPsY32uOPqEjaisQScnzO/XvlhQTzj5w+hFObjiNgIaHRceYh3hZZwsRsHIkCxOY0JgUPeFP9IVXso0VptIjCPRF5yrV/+dF1rtl4eyYj/XOBvSDzbQQwqdjhHffw0TXW0f/yjGGJCYM+tw/9dmj9VilAMNTx1H76uPKUo4M3vLBQLo2tj7z1jlh4Jlw5hKBRcWQWbpWP95p71Db6gSpqReDYbx57BW19APMVketUYsXfXTztM/HWz35J9HDya3ID0Dl+pE22Wo8SZo2+ULKu/4OYVcD8DjF15WwXrcuFDypX132j+LUWOVWxCs5hdMybSDwF3ZhVBH ec2-user@ip-172-31-41-191.eu-west-1.compute.internal" # NOQA
+TEST_MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/")
+
def random_string(
size=9,
@@ -91,12 +93,11 @@ def tmp_app(request):
}
},
"SECRET_KEY": "secret",
- "FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
- "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "RETRIEVE_MONGO_URI": TEST_MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db_name,
"RETRIEVE_MONGO_COLLECTION": "datasets",
- "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "SEARCH_MONGO_URI": TEST_MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db_name,
"SEARCH_MONGO_COLLECTION": "datasets",
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
@@ -121,7 +122,9 @@ def tmp_app(request):
@request.addfinalizer
def teardown():
current_app.retrieve.client.drop_database(tmp_mongo_db_name)
+ current_app.retrieve.client.close()
current_app.search.client.drop_database(tmp_mongo_db_name)
+ current_app.search.client.close()
sql_db.session.remove()
return app
@@ -152,12 +155,11 @@ def tmp_app_with_users(request):
"API_VERSION": 'v1',
"OPENAPI_VERSION": '3.0.2',
"SECRET_KEY": "secret",
- "FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
- "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "RETRIEVE_MONGO_URI": TEST_MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db_name,
"RETRIEVE_MONGO_COLLECTION": "datasets",
- "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "SEARCH_MONGO_URI": TEST_MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db_name,
"SEARCH_MONGO_COLLECTION": "datasets",
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
@@ -198,7 +200,9 @@ def tmp_app_with_users(request):
@request.addfinalizer
def teardown():
current_app.retrieve.client.drop_database(tmp_mongo_db_name)
+ current_app.retrieve.client.close()
current_app.search.client.drop_database(tmp_mongo_db_name)
+ current_app.search.client.close()
sql_db.session.remove()
return app
@@ -228,12 +232,11 @@ def tmp_app_with_data(request):
"API_TITLE": 'dservercore API',
"API_VERSION": 'v1',
"OPENAPI_VERSION": '3.0.2',
- "FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
- "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "RETRIEVE_MONGO_URI": TEST_MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db_name,
"RETRIEVE_MONGO_COLLECTION": "datasets",
- "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "SEARCH_MONGO_URI": TEST_MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db_name,
"SEARCH_MONGO_COLLECTION": "datasets",
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
@@ -331,7 +334,9 @@ def tmp_app_with_data(request):
@request.addfinalizer
def teardown():
current_app.retrieve.client.drop_database(tmp_mongo_db_name)
+ current_app.retrieve.client.close()
current_app.search.client.drop_database(tmp_mongo_db_name)
+ current_app.search.client.close()
sql_db.session.remove()
return app
@@ -355,12 +360,11 @@ def tmp_cli_runner(request):
"API_TITLE": 'dservercore API',
"API_VERSION": 'v1',
"OPENAPI_VERSION": '3.0.2',
- "FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
- "RETRIEVE_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "RETRIEVE_MONGO_URI": TEST_MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db_name,
"RETRIEVE_MONGO_COLLECTION": "datasets",
- "SEARCH_MONGO_URI": os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
+ "SEARCH_MONGO_URI": TEST_MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db_name,
"SEARCH_MONGO_COLLECTION": "datasets",
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
@@ -381,7 +385,9 @@ def tmp_cli_runner(request):
@request.addfinalizer
def teardown():
current_app.retrieve.client.drop_database(tmp_mongo_db_name)
+ current_app.retrieve.client.close()
current_app.search.client.drop_database(tmp_mongo_db_name)
+ current_app.search.client.close()
sql_db.session.remove()
return app.test_cli_runner()