diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4b8f6fb..4f8a3b7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -106,7 +106,7 @@ updates: - patch # Python conformance + crypt4gh-interop tooling (conformance/requirements*.txt). - # CI-only (pyshacl/rdflib/ckanext-dcat for the FDP conformance job; crypt4gh for + # CI-only (pyshacl/rdflib for the FDP conformance job; crypt4gh for # the interop oracle) and never shipped, but still fetched-and-executed on the # runner — so it gets security updates like every other dependency input. The # top-level pins are exact; regenerate the lock with a security PR (see the header), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77f1667..f6ba0dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,7 @@ jobs: # `ring`). All defined in scripts/ci-local.sh. - run: ./scripts/ci-local.sh profiles - # FDP conformance — Python (pySHACL + ckanext-dcat), isolated from the Rust + # FDP conformance — Python (pySHACL + harvest-profile reads), isolated from the Rust # build above and never shipped in the binaries. Builds the crawl producer, # which ingests the fixture # corpus, crawls the live FDP harvester-style, unions the Turtle, and runs @@ -210,7 +210,7 @@ jobs: "$RUNNER_TEMP/audit-venv/bin/pip-audit" --strict --no-deps -r conformance/requirements.lock - name: Build the crawl producer + agreement test run: cargo test -p gdi-node-standalone --locked --no-run --test conformance_crawl --test conformance_agreement - - name: pySHACL + ckanext-dcat conformance over the crawled-and-unioned FDP graph + - name: pySHACL + harvest-profile conformance over the crawled-and-unioned FDP graph env: GDI_FDP_PYTHON: ${{ runner.temp }}/fdp-venv/bin/python run: | diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 644f26d..3a66d8d 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -223,24 +223,19 @@ jobs: # (conformance/shapes/gdi-metadata/, conformance/ga4gh-beacon-v2/, # conformance/ga4gh-beacon-v2-default-model/, conformance/ga4gh-vrs-1.3/) from the commit # pinned in its VENDORED.md and asserts it is still byte-identical — turning the - # "verbatim at commit X" claim from documentation into an enforced check. `check` also - # runs the `pins` assertions: the userportal's deployed profile code, at the tags its own - # ckan/Dockerfile names, and the beacon-v2 release watch (so a consumer change surfaces - # here as a ticket, not a silent harvest break). It is off the per-PR `ci` workflow because - # it needs network access and gates nothing on a PR. A red run means a vendored file was - # locally edited, or upstream content or a pinned token changed — re-vendor with - # `scripts/vendored.sh fetch` + bump VENDORED.md, re-check and re-pin in scripts/vendored.sh, - # or revert the edit. + # "verbatim at commit X" claim from documentation into an enforced check. External pins + # run only in the `pins` job below. It is off the per-PR `ci` workflow because it needs + # network access and gates nothing on a PR. Red means a vendored file was edited locally + # (re-vendor with `scripts/vendored.sh fetch` and bump VENDORED.md, or revert the edit) or + # upstream could not be reached. # (The authored FDP shapes under conformance/shapes/fdp/ are not vendored — see - # their PROVENANCE.md — so they are not byte-checked here.) GITHUB_TOKEN raises the - # unauthenticated rate limit for the raw fetches. + # their PROVENANCE.md — so they are not byte-checked here.) Both steps fetch from + # raw.githubusercontent, which needs no token. vendored-sync: runs-on: ubuntu-26.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Verify vendored files + external upstream pins match their pins - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Verify vendored files match their pinned commits run: ./scripts/vendored.sh check # ...and the question `check` structurally cannot answer. It fetches each set at its # pinned commit — an immutable SHA — so it stays green no matter how far upstream @@ -255,8 +250,6 @@ jobs: # below runs `drift gdi-metadata` fatally. - name: Report whether upstream has moved past our pinned commits continue-on-error: true - env: - GITHUB_TOKEN: ${{ github.token }} run: ./scripts/vendored.sh drift # macOS + Windows cross-compile of the gdi-dataset-tool CLI — moved off the per-PR @@ -498,14 +491,13 @@ jobs: key: gdi-corpus-${{ hashFiles('scripts/fetch-corpus.sh') }} - run: ./scripts/ci-local.sh corpus - # External pin freshness: the userportal's deployed profile code, the vendored gdi-metadata - # shapes against upstream, the beacon-v2 release and action tags, plus the distroless - # runtime digest in the Dockerfile. Weekly, not per-PR, because it is a verdict about the - # world rather than about the commit: it goes red when a consumer contract moves, which no - # pull request caused, and a gate that reddens for reasons the author cannot fix is a gate - # people learn to ignore. An unreachable upstream is a warning. A rebuilt distroless base - # is a note, not a failure: `image-scan` above already fails on a fixable CVE in the pinned - # one. + # External pin freshness: the vendored gdi-metadata shapes against upstream, the beacon-v2 + # release and action tags, plus the distroless runtime digest in the Dockerfile. Weekly, + # not per-PR, because it is a verdict about the world rather than about the commit: it + # goes red when a consumer contract moves, which no pull request caused, and a gate that + # reddens for reasons the author cannot fix is a gate people learn to ignore. An + # unreachable upstream is a warning. A rebuilt distroless base is a note, not a failure: + # `image-scan` above already fails on a fixable CVE in the pinned one. pins: runs-on: ubuntu-26.04 steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 365789a..e8cbb32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -535,7 +535,7 @@ checks each pinned SHA against its comment upstream. | **`sbom`** | `cargo cyclonedx --all --format json` over the whole `Cargo.lock`, producing a CycloneDX SBOM artifact. | | **`cross-compile`** | Five x86_64-Linux legs via `cross`, on the `release-verify` profile: the `gdi-dataset-tool` CLI for gnu and static musl, the service for gnu and static musl at `--features full`, and a musl `--features s3` build-verify. The aarch64 and macOS/Windows legs run weekly. | | **`e2e-smoke`** | `./scripts/e2e/run.sh`: build the service image, bring up the minimal Compose stack, pack a fixture, poll `/datasets/{id}/state` to `visible`, then assert a Beacon query, an FDP crawl, and publish/unpublish. | -| **`conformance`** | pySHACL and ckanext-dcat over the crawled-and-unioned FDP graph, plus the dual-encoding agreement test. Python only, isolated from the Rust build and never shipped. | +| **`conformance`** | pySHACL and the harvest-profile read check (`check_ckanext.py`) over the crawled-and-unioned FDP graph, plus the dual-encoding agreement test. Python only, isolated from the Rust build and never shipped. | | **`crypt4gh-interop`** | Rust↔Python crypt4gh round-trip (encrypt→decrypt, reverse, key parse) against the pinned `crypt4gh==1.8.6`. Isolated and never shipped. | | **`secrets`** | `gitleaks` over the working tree against the repo's own `.gitleaks.toml`, so the same exemptions govern CI and a developer machine. gitleaks is fetched from its release and checksum-verified, because it is not in install-action's tool list. | | **`fuzz-smoke`** | Compile-gates every `cargo-fuzz` target against the current library API, on the pinned stable toolchain — it does not fuzz. Also runs the pin- and licence-drift comparisons between the isolated fuzz manifest and the root workspace, which nothing else checks. | @@ -545,13 +545,11 @@ checks each pinned SHA against its comment upstream. The `pins` leg has no `ci.yml` job: it is a verdict about the world rather than about the commit, so it runs weekly in `scheduled.yml` as its own `pins` job (with `PINS_STRICT=1`), alongside `vendored-sync`. -`scripts/vendored.sh pins` asserts that the userportal's deployed profile code (at the tags -its own `ckan/Dockerfile` names, so a userportal release that moves a tag without moving the -profile passes), the GA4GH beacon-v2 latest release tag and the pinned GitHub Action tags -still resolve to what this tree pins; `vendored.sh drift gdi-metadata` compares the vendored -shapes against upstream `main`; and `ci-local.sh` itself compares the Dockerfile's distroless -runtime digest against upstream, through Docker. It needs network, and unauthenticated -GitHub API calls are rate-limited per IP, so export `GITHUB_TOKEN` — sent to +`scripts/vendored.sh pins` asserts that the GA4GH beacon-v2 latest release tag and the pinned +GitHub Action tags still resolve to what this tree pins; `vendored.sh drift gdi-metadata` +compares the vendored shapes against upstream `main`; and `ci-local.sh` itself compares the +Dockerfile's distroless runtime digest against upstream, through Docker. It needs network, and +unauthenticated GitHub API calls are rate-limited per IP, so export `GITHUB_TOKEN` — sent to `api.github.com` only — to lift the budget. Real drift, including a pinned path that 404s, warns in `pins` and fails under `PINS_STRICT=1`, which `pins-strict`, `release` and the weekly job set. A rebuilt base image is a note unless `BASE_IMAGE_STRICT=1`, which only `pins-strict` @@ -612,12 +610,11 @@ inside the `release` meta-leg are marked. rather than per-PR because the leg range-downloads those slices first; the corpus is cached between runs. Local: `ci-local.sh corpus`. -- **`pins`** — external pin freshness: the userportal's deployed profile code, the vendored - gdi-metadata shapes against upstream, the beacon-v2 release and action tags, plus the - distroless runtime digest in the Dockerfile (a note there). Weekly because it is a - verdict about the world, not about the commit. Local: - `PINS_STRICT=1 ci-local.sh pins` (the job sets that so drift is fatal there; a working - tree only warns). +- **`pins`** — external pin freshness: the vendored gdi-metadata shapes against upstream, + the beacon-v2 release and action tags, plus the distroless runtime digest in the + Dockerfile (a note there). Weekly because it is a verdict about the world, not about the + commit. Local: `PINS_STRICT=1 ci-local.sh pins` (the job sets that so drift is fatal + there; a working tree only warns). - **`notify`** — opens or updates one tracking issue on a failed weekly run. No local equivalent. - **`e2e-smoke-full`** (`e2e-full.yml`) — the full Garage + OpenBao + PME crypt4gh diff --git a/Cargo.lock b/Cargo.lock index e743461..e476d70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3103,9 +3103,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 4fce63f..081a440 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,7 +206,7 @@ reqwest = { version = "=0.13.5", default-features = false, features = ["json", " # rustls with `custom-provider`, which disables the implicit provider so one must be # installed explicitly. Shared by the service's `tls` group and the tool's startup # crypto-provider install; this is the surface that keeps the build on `ring` alone. -rustls = { version = "=0.23.45", default-features = false, features = ["ring", "custom-provider", "logging", "tls12"] } +rustls = { version = "=0.23.46", default-features = false, features = ["ring", "custom-provider", "logging", "tls12"] } # Mozilla CA roots as full DER certificates: `webpki-root-certs`, not the trust-anchor # `webpki-roots`, which reqwest 0.13 no longer consumes. Merged on top of the host platform # trust store, via reqwest's `tls_certs_merge` and object_store's `with_root_certificate`, diff --git a/THIRD-PARTY-LICENSES.md b/THIRD-PARTY-LICENSES.md index 5f6c359..6d4d06a 100644 --- a/THIRD-PARTY-LICENSES.md +++ b/THIRD-PARTY-LICENSES.md @@ -6150,7 +6150,7 @@ Used by: - ring 0.17.14 - rustix 1.1.5 - rustls-native-certs 0.8.4 -- rustls 0.23.45 +- rustls 0.23.46 - scopeguard 1.2.0 - security-framework-sys 2.17.0 - security-framework 3.7.0 diff --git a/conformance/README.md b/conformance/README.md index c41622e..029784e 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -104,23 +104,18 @@ into `target/conformance/union.ttl`. It then runs: localized paths, and both are exercised: the enriched corpus dataset is bilingual in both, while the minimal one stays plain so the `LocalizedText::Plain` render path is covered too. -- **`check_ckanext.py `** — consumer compatibility, the nice-to-have check. +- **`check_ckanext.py `** — consumer compatibility. The profile the userportal deploys is `fairdatapoint_dcat_ap` (`ckanext.fairdatapoint.profiles.FAIRDataPointDCATAPProfile`), which subclasses `euro_health_dcat_ap` to add tag validation, `tags_translated` sanitising and label resolution. It overrides no predicate read, so the parent's predicate set is the deployed one. It ships in the harvester extension - (`gdi-userportal-ckanext-fairdatapoint`), not in `ckanext-dcat`, so a harvester bump can - change the profile with the `ckanext-dcat` pin standing still; the profile code of both - forks is watched by content, at the tags the userportal deploys (see - [Version pins](#version-pins)). - - The check prefers the real `ckanext-dcat` `RDFParser` on the parent profile, since the - subclass's package is not installed in this venv and only the parent's reads are under - test. The standalone venv has no full CKAN, so it degrades to a direct rdflib parse - mirroring those exact predicate reads and prints a `[best-effort]` note. In practice that - fallback is the live tier: `ckanext-dcat==2.4.4` installs, but importing its `RDFParser` - raises `ModuleNotFoundError: No module named 'ckan'`, since the venv has no CKAN. + (`gdi-userportal-ckanext-fairdatapoint`), not in `ckanext-dcat`. + + The real `RDFParser` needs a full CKAN install, so the check mirrors its predicate reads + in plain rdflib. Its docstring names the fork versions the list was last checked + against. CI does not watch the forks, so recheck the list if a harvest of this node + loses a field. Every field is required on every dataset, not merely somewhere in the graph. An existential check would let one healthy dataset vouch for all of them, which matters here @@ -197,19 +192,9 @@ You can also run a check by hand once a `union.ttl` exists: the gdi-metadata model tag the node's static mapping table encodes. The vendored shapes under `shapes/gdi-metadata/` are that model; see `shapes/gdi-metadata/VENDORED.md` for the exact upstream commit they were copied from. -- **pyshacl / rdflib / ckanext-dcat** — pinned in `requirements.txt` and resolved into the +- **pyshacl / rdflib** — pinned in `requirements.txt` and resolved into the hash-locked `requirements.lock` (regenerate both together with `uv pip compile --universal --generate-hashes`); bump them in a reviewed change. - `ckanext-dcat` is pinned to `2.4.4`, the upstream base of the GDI fork the userportal - deploys (`gdi-userportal-ckanext-dcat`, with the harvester - `gdi-userportal-ckanext-fairdatapoint`, at the tags gdi-userportal-ckan-docker's - `ckan/Dockerfile` names). The profile the deployment's harvest sources name is - `fairdatapoint_dcat_ap`, supplied by the harvester as a subclass of the fork's - `euro_health_dcat_ap`, so the `ckanext-dcat` pin alone would not see a profile change. - That is why both profile files are watched below by content at the deployed tag, and the - fork's `pyproject.toml` version against this `2.4.4` pin. `check_ckanext.py` runs an - rdflib hand-mirror of the inherited `euro_health_dcat_ap` reads, because the real - `RDFParser` needs full CKAN and so never loads in this venv. - **Guards on all of the above.** Which gate reaches each one differs, and the difference matters, so it is stated per guard rather than claimed for the list. The offline integrity of these files is fully covered by `scripts/ci-local.sh all`; the two guards @@ -220,21 +205,16 @@ You can also run a check by hand once a `union.ttl` exists: the dangerous direction, because weakening a vendored `sh:minCount` or deleting a `"required"` array from a Beacon schema makes everything downstream pass more easily. - `vendored.sh pins` (network, in the `pins` leg). Under `all`, both an unreachable - network and real drift warn and pass: drift here means the federation moved, not that - this tree broke, so it is reported rather than fatal. The weekly job, `release` and - `pins-strict` set `PINS_STRICT=1` and fail closed on drift. Two watches: - `USERPORTAL_PROFILE_PINS`, the deployed profile files by hash and the dcat fork's - upstream base against this `requirements.txt`, resolved at the tags the userportal's - own `ckan/Dockerfile` names, so a userportal release that moves a tag but neither fact - passes; and `EXTERNAL_PINS`, the latest GA4GH beacon-v2 release tag — the only watch on - a standard rather than on what the federation deploys. That one reads the releases API, - because a release is a tag and tags are in no file the repo contains: beacon-v2's own - CHANGELOG tops out at `2.0.0` while the repo is tagged `v2.2.0`, so a file-based watch - would be a dead guard. The same subcommand also runs `check_action_pins`, which is - about this repo rather than the federation: it re-resolves every - `uses: owner/repo@ # ` in the workflows and reports a SHA that no longer is - that tag. The `pins` leg also runs `vendored.sh drift gdi-metadata` (below), with the - same warn-or-fail split. + network and real drift warn and pass: drift here means upstream moved, not that this + tree broke, so it is reported rather than fatal. The weekly job, `release` and + `pins-strict` set `PINS_STRICT=1` and fail closed on drift. It watches `EXTERNAL_PINS`, + the latest GA4GH beacon-v2 release tag. That reads the releases API, because a release + is a tag and tags are in no file the repo contains: beacon-v2's own CHANGELOG tops out + at `2.0.0` while the repo is tagged `v2.2.0`, so a file-based watch would be a dead + guard. The same subcommand also runs `check_action_pins`, which is about this repo's own + pins: it re-resolves every `uses: owner/repo@ # ` in the workflows and reports + a SHA that no longer is that tag. The `pins` leg also runs + `vendored.sh drift gdi-metadata` (below), with the same warn-or-fail split. - `pip-audit` (in the `supply_chain` leg) — the advisory scan for the one corner `cargo deny` cannot see. It audits the `.lock` files, not the `.txt`: pip-audit resolves a requirements file, and the loose `.txt` can resolve a newer version than the lock pins @@ -271,9 +251,9 @@ You can also run a check by hand once a `union.ttl` exists: when you touch a vendored input: `scripts/vendored.sh check` and `scripts/vendored.sh drift`, or a full `scripts/ci-local.sh release`. - One input has no automated watch: `shapes/fdp/` is hand-derived from the FDP v1.2 - specification document, and no watch on a spec document would be meaningful. See its - `PROVENANCE.md`. + Two inputs have no automated watch: `shapes/fdp/`, hand-derived from the FDP v1.2 + specification (see its `PROVENANCE.md`), and the userportal's harvest profile, which + `check_ckanext.py` mirrors by hand. ## Beacon wire-schema conformance diff --git a/conformance/check_ckanext.py b/conformance/check_ckanext.py index f1a1565..edf31e8 100755 --- a/conformance/check_ckanext.py +++ b/conformance/check_ckanext.py @@ -2,9 +2,8 @@ """Consumer-compatibility check: does the node's FDP graph survive the reads the userportal's DCAT consumer performs? -This is the nice-to-have conformance check, complementing ``check_fdp.py`` (pySHACL, the -must-have). pySHACL checks the shape; this checks what is actually read, catching -namespace and predicate mismatches a passing SHACL shape would not. +Complements ``check_fdp.py``: SHACL checks the shape, this checks what the harvester +actually reads, which catches namespace and predicate mismatches a passing shape would not. The deployed profile is ``fairdatapoint_dcat_ap`` (``ckanext.fairdatapoint.profiles.FAIRDataPointDCATAPProfile``), a subclass of @@ -14,31 +13,16 @@ the harvester repository (``gdi-userportal-ckanext-fairdatapoint``), not in ``ckanext-dcat``. It adds tag validation, ``tags_translated`` sanitising and label resolution on top of the parent's reads and overrides no predicate, so the parent's -predicate set below is the deployed one. A harvester bump therefore changes this profile -without touching the ``ckanext-dcat`` pin, which is why both forks' profile files are -watched by content, at the tags the userportal deploys, by ``vendored.sh pins`` (see -``conformance/README.md``). - -In practice the rdflib mirror (tier 2) is what runs: the real ``ckanext-dcat`` -``RDFParser`` needs full CKAN (``ckantoolkit`` imports ``ckan``), which the isolated -conformance venv does not provide, so tier 1 does not load here. The hand-mirror is -therefore the live check, pinned to the fork the userportal deploys, rather than a -degraded stand-in. Tier 1 stays wired for a venv that does carry full CKAN. - -Two tiers, in order of fidelity: - -1. Real parser (preferred): round-trip the union through - ``ckanext.dcat.processors.RDFParser`` with the ``euro_health_dcat_ap`` profile. That - is the deployed profile's parent, and the closest this venv can get: the subclass - ships in ``ckanext-fairdatapoint``, which is not installed here (only ``ckanext-dcat`` - is pinned in ``requirements.txt``), and the reads under test are the parent's. -2. Best-effort fallback: when that import fails, parse the union directly with rdflib, - mirroring the deployed profile's exact predicate reads — those of - ``EuropeanHealthDCATAPProfile``, since the subclass adds none (DCAT-AP 3.0 core plus - the ``healthdcatap:`` / ``dpv:`` predicate set) — and assert the same fields survive. - It prints a ``[best-effort]`` note and the tier used. - -Either way the assertions are the same consumer-facing fields: a ``dcat:Catalog``; each +predicate set below is the deployed one. + +The real ``RDFParser`` needs a full CKAN install, so this check mirrors its predicate reads +in plain rdflib. The list was last checked on 2026-10-19 against the forks the userportal +then deployed (``gdi-userportal-ckanext-dcat`` v2.5.2, +``gdi-userportal-ckanext-fairdatapoint`` v1.7.3). CI does not watch them, so if a harvest +of this node loses a field, compare this list with the dcat fork's +``profiles/euro_health_dcat_ap.py`` and ``profiles/euro_dcat_ap_base.py``. + +The assertions are the consumer-facing fields: a ``dcat:Catalog``; each ``dcat:Dataset``'s ``dct:title``, ``dct:identifier`` and ``dct:language``; each ``dcat:Distribution``'s ``dcat:accessURL`` and ``dct:format``; and the healthdcatap fields the profile reads (``healthdcatap:numberOfRecords``, ``healthdcatap:healthCategory``, @@ -46,9 +30,8 @@ and ``dct:format`` — are mandated by no SHACL shape, so this check is their only per-record cover. -Exit code: 0 on a successful parse or a clean skip, so a missing CKAN never hard-fails the -build (pySHACL is the gate); non-zero only when a field that should have survived the -parse is missing. +Exits 0 when every dataset carries every field, 1 when one is missing, 2 on bad +arguments. Usage:: @@ -78,12 +61,12 @@ def _require(condition: bool, message: str) -> None: raise CheckFailure(message) -def _require_every(missing: dict[str, list[str]], total: int, tier: str) -> None: +def _require_every(missing: dict[str, list[str]], total: int) -> None: """Fail if any dataset is missing a consumer-facing field. The quantifier matters. An existential check over the whole graph would let a single surviving dataset with a ``dct:title`` vouch for every dataset, which is wrong for a - consumer-compatibility check and worst exactly where this tier is the only cover: + consumer-compatibility check and worst exactly where this check is the only cover: ``healthdcatap:hdab`` and ``healthdcatap:numberOfRecords`` are mandated by no SHACL shape, so nothing else in the suite looks at them. An existential check stays green until the field is stripped from every dataset at once. @@ -94,106 +77,14 @@ def _require_every(missing: dict[str, list[str]], total: int, tier: str) -> None f"{field}: missing from {len(ids)} of {total} dataset(s) (e.g. {min(ids)})" for field, ids in sorted(missing.items()) ) - raise CheckFailure( - f"[{tier}] consumer-facing field(s) absent per-dataset — {detail}" - ) - - -def try_real_parser(union_path: Path) -> bool: - """Attempt the real ``ckanext-dcat`` ``RDFParser`` + health profile. - - Returns True if it ran (and the assertions passed); False if ``ckanext-dcat`` - cannot be wired up standalone (missing CKAN) so the caller should fall back. - Raises :class:`CheckFailure` if it ran but an expected field was dropped. - """ - try: - from ckanext.dcat.processors import RDFParser - # BLE001: the blind catch is deliberate. This probe must degrade to the best-effort - # rdflib parse on any failure to wire up ckanext-dcat standalone: ImportError without - # CKAN, but also the AttributeError/RuntimeError its plugin loader raises part-way in. - # Narrowing it would turn a tier downgrade into a conformance-leg crash. - except Exception as exc: # noqa: BLE001 # pragma: no cover - depends on env - print( - f"[best-effort] real ckanext-dcat RDFParser unavailable ({type(exc).__name__}: {exc}); " - "falling back to a direct rdflib consumer-parse." - ) - return False - - # The deployed profile's PARENT: `fairdatapoint_dcat_ap` subclasses this one and - # ships in `ckanext-fairdatapoint`, which this venv does not install (see the module - # docstring). The predicate reads asserted below are the parent's either way. - parser = RDFParser(profiles=["euro_health_dcat_ap"]) - parser.parse(union_path.read_text(encoding="utf-8"), _format="turtle") - datasets = list(parser.datasets()) - _require(bool(datasets), "ckanext-dcat parsed zero datasets from the FDP graph") - - # Per-dataset, and each healthdcatap field separately: accepting `health_category OR - # number_of_records OR hdab` as one flag would make this tier weaker than the rdflib - # fallback below, which requires all three, so coverage would silently drop the day a - # venv did carry full CKAN. Keep the two tiers asserting the same thing. - missing: dict[str, list[str]] = {} + raise CheckFailure(f"consumer-facing field(s) absent per-dataset — {detail}") - def note(field: str, ident: str) -> None: - missing.setdefault(field, []).append(ident) - for ds in datasets: - ident = str(ds.get("identifier") or ds.get("name") or ds.get("title") or "?") - if not ds.get("title"): - note("title", ident) - if not ds.get("identifier"): - note("identifier", ident) - resources = ds.get("resources", []) - if not resources or not all( - r.get("access_url") or r.get("url") for r in resources - ): - note("accessURL", ident) - # The CKAN-dict twins of the two predicates tier 2 checks below. Key names read - # off the INSTALLED `ckanext/dcat/profiles/euro_dcat_ap_base.py`: the distribution - # loop sets `resource_dict["format"]` from `_distribution_format`, and the dataset - # list loop maps `dct:language` to `language`. - # Weaker than tier 2's `dct:format` check: `_distribution_format` also populates - # `format` from a `dcat:mediaType` fallback, so this passes on a distribution - # carrying only mediaType. It is inert while tier 1 does not run; tighten it - # against the installed profile's keys once a full-CKAN venv exists. - if not resources or not all(r.get("format") for r in resources): - note("format", ident) - for key in ("language", "health_category", "number_of_records", "hdab"): - if not ds.get(key): - note(key, ident) - - _require_every(missing, len(datasets), "real parser") - print( - f"[real parser] ckanext-dcat euro_health_dcat_ap parsed {len(datasets)} dataset(s); " - "title / identifier / accessURL / format / language / healthdcatap fields survived " - "on every one." - ) - return True - - -def fallback_direct_parse(g: Graph) -> None: +def direct_parse(g: Graph) -> None: """Direct rdflib parse mirroring the deployed profile's predicate reads. - Asserts the same consumer-facing fields survive, using the exact predicates - ``EuropeanHealthDCATAPProfile.parse_dataset`` reads (verified against the - installed ``ckanext/dcat/profiles/euro_health_dcat_ap.py`` and ``euro_dcat_ap_3``). - - Pins: the deployed profile is ``fairdatapoint_dcat_ap`` from - ``gdi-userportal-ckanext-fairdatapoint``, a subclass that adds tag sanitising and label - resolution and overrides no predicate read, so this hand-mirror tracks its parent in the - GDI fork ``gdi-userportal-ckanext-dcat`` (what the userportal deploys; - ``requirements.txt`` pins the fork's upstream base ``ckanext-dcat==2.4.4`` for the - optional real-parser tier). ``vendored.sh pins`` watches both profile files by hash at - the tags the userportal's ``ckan/Dockerfile`` names, because a harvester bump can change - the profile with the ``ckanext-dcat`` pin standing still. - - The parent ``euro_health_dcat_ap`` profile reads - ``healthdcatap:{numberOfRecords,healthCategory,hdab}`` (namespace - ``http://healthdataportal.eu/ns/health#``) and ``dpv:hasLegalBasis`` - (``https://w3id.org/dpv#``), plus the DCAT-AP core ``dct:title``, ``dct:identifier`` - and ``dcat:accessURL`` — the subset asserted below, checked against the fork's - ``profiles/euro_health_dcat_ap.py`` at the hash ``USERPORTAL_PROFILE_PINS`` pins. - Re-check this list when that hash drifts, so the best-effort tier cannot drift from the - deployed profile. + Asserts that every dataset carries the fields the module docstring lists. + ``dpv:hasLegalBasis`` is optional, so it is only counted. """ catalogs = list(g.subjects(RDF.type, DCAT.Catalog)) _require( @@ -246,22 +137,18 @@ def note(field: str, subject: object) -> None: if next(g.objects(ds, pred), None) is None: note(label, ds) - _require_every(missing, len(datasets), "best-effort") + _require_every(missing, len(datasets)) # dpv:hasLegalBasis is read as dpv namespace, never healthdcatap. # It is optional per-dataset, so we only note its presence, not require it. legal = list(g.objects(predicate=DPV.hasLegalBasis)) print( - f"[best-effort] direct rdflib consumer-parse of {len(datasets)} dataset(s), " + f"rdflib consumer-parse of {len(datasets)} dataset(s), " f"{len(catalogs)} catalog(s): title / identifier / accessURL / format / language / " "healthdcatap:{numberOfRecords,healthCategory,hdab} present on every dataset" + (f"; dpv:hasLegalBasis seen {len(legal)}x" if legal else "") + "." ) - print( - "[best-effort] note: the standalone conformance venv has no full CKAN, so the actual " - "ckanext-dcat RDFParser could not run; this mirrors its exact predicate reads instead." - ) def main(argv: list[str]) -> int: @@ -274,13 +161,10 @@ def main(argv: list[str]) -> int: return 2 try: - if try_real_parser(union_path): - print("\nckanext-dcat consumer-compatibility: OK (real parser).") - return 0 g = Graph() g.parse(union_path, format="turtle") - fallback_direct_parse(g) - print("\nckanext-dcat consumer-compatibility: OK (best-effort fallback).") + direct_parse(g) + print("\nckanext-dcat consumer-compatibility: OK.") return 0 except CheckFailure as exc: print(f"\nckanext-dcat consumer-compatibility FAILED: {exc}", file=sys.stderr) diff --git a/conformance/requirements.lock b/conformance/requirements.lock index ae1717d..0e5b73d 100644 --- a/conformance/requirements.lock +++ b/conformance/requirements.lock @@ -1,299 +1,9 @@ # This file was autogenerated by uv via the following command: # uv pip compile --universal --generate-hashes conformance/requirements.txt -o conformance/requirements.lock -cachetools==7.2.1 \ - --hash=sha256:63aa53dfe7473c10cccdd5a01dedf76ef2c4b73a58840d9396e7d0752cbdac3b \ - --hash=sha256:b1a7537025c06abf96fcc1443e496af9a3fb95e774e70e1f0af226f73f7f2dcc - # via pyld -ckanext-dcat==2.4.4 \ - --hash=sha256:cb862017ff70622fd39e576e1c2db928b637dee0ca4186ff2c441236fe46242d \ - --hash=sha256:d291b0d8c2e8112e86966d6c5a591696f155ef19dfea76d3cc78434d5dfa37ba - # via -r conformance/requirements.txt -ckantoolkit==0.0.7 \ - --hash=sha256:72432c1a1c48b96a6ebc614f11fc2b9c3c27d7507dbbd35534068ef060a2c754 - # via ckanext-dcat -click==8.5.0 \ - --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ - --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 - # via geomet -frozendict==2.4.7 \ - --hash=sha256:05dd27415f913cd11649009f53d97eb565ce7b76787d7869c4733738c10e8d27 \ - --hash=sha256:0664092614d2b9d0aa404731f33ad5459a54fe8dab9d1fd45aa714fa6de4d0ef \ - --hash=sha256:0ece525da7d0aa3eb56c3e479f30612028d545081c15450d67d771a303ee7d4c \ - --hash=sha256:0ff6f57854cc8aa8b30947ec005f9246d96e795a78b21441614e85d39b708822 \ - --hash=sha256:115a822ecd754574e11205e0880e9d61258d960863d6fd1b90883aa800f6d3b3 \ - --hash=sha256:11d35075f979c96f528d74ccbf89322a7ef8211977dd566bc384985ebce689be \ - --hash=sha256:1662f1b72b4f4a2ffdfdc4981ece275ca11f90244208ac1f1fc2c17fc9c9437a \ - --hash=sha256:176a66094428b9fd66270927b9787e3b8b1c9505ef92723c7b0ef1923dbe3c4a \ - --hash=sha256:176dd384dfe1d0d79449e05f67764c57c6f0f3095378bf00deb33165d5d2df5b \ - --hash=sha256:1c521ad3d747aa475e9040e231f5f1847c04423bae5571c010a9d969e6983c40 \ - --hash=sha256:1df8e22f7d24172c08434b10911f3971434bb5a59b4d1b0078ae33a623625294 \ - --hash=sha256:1e307be0e1f26cbc9593f6bdad5238a1408a50f39f63c9c39eb93c7de5926767 \ - --hash=sha256:1e801d62e35df24be2c6f7f43c114058712efa79a8549c289437754dad0207a3 \ - --hash=sha256:2808bab8e21887a8c106cca5f6f0ab5bda7ee81e159409a10f53d57542ccd99c \ - --hash=sha256:294a7d7d51dd979021a8691b46aedf9bd4a594ce3ed33a4bdf0a712d6929d712 \ - --hash=sha256:2b96f224a5431889f04b2bc99c0e9abe285679464273ead83d7d7f2a15907d35 \ - --hash=sha256:2cf0a665bf2f1ce69d3cd8b6d3574b1d32ae00981a16fa1d255d2da8a2e44b7c \ - --hash=sha256:2e5d2c30f4a3fea83a14b0a5722f21c10de5c755ab5637c70de5eb60886d58cd \ - --hash=sha256:2ebd953c41408acfb8041ff9e6c3519c09988fb7e007df7ab6b56e229029d788 \ - --hash=sha256:313e0e1d8b22b317aa1f7dd48aec8cbb0416ddd625addf7648a69148fcb9ccff \ - --hash=sha256:34233deb8d09e798e874a6ac00b054d2e842164d982ebd43eb91b9f0a6a34876 \ - --hash=sha256:346a53640f15c1640a3503f60ba99df39e4ab174979f10db4304bbb378df5cbd \ - --hash=sha256:3842cfc2d69df5b9978f2e881b7678a282dbdd6846b11b5159f910bc633cbe4f \ - --hash=sha256:39abe54264ae69a0b2e00fabdb5118604f36a5b927d33e7532cd594c5142ebf4 \ - --hash=sha256:3ed9e2f3547a59f4ef5c233614c6faa6221d33004cb615ae1c07ffc551cfe178 \ - --hash=sha256:48ab42b01952bc11543577de9fe5d9ca7c41b35dda36326a07fb47d84b3d5f22 \ - --hash=sha256:4c64d34b802912ee6d107936e970b90750385a1fdfd38d310098b2918ba4cbf2 \ - --hash=sha256:5694417864875ca959932e3b98e2b7d5d27c75177bf510939d0da583712ddf58 \ - --hash=sha256:57134ef5df1dd32229c148c75a7b89245dbdb89966a155d6dfd4bda653e8c7af \ - --hash=sha256:57a754671c5746e11140363aa2f4e7a75c8607de6e85a2bf89dcd1daf51885a7 \ - --hash=sha256:5943c3f683d3f32036f6ca975e920e383d85add1857eee547742de9c1f283716 \ - --hash=sha256:5c1781f28c4bbb177644b3cb6d5cf7da59be374b02d91cdde68d1d5ef32e046b \ - --hash=sha256:6991469a889ee8a108fe5ed1b044447c7b7a07da9067e93c59cbfac8c1d625cf \ - --hash=sha256:6d30dbba6eb1497c695f3108c2c292807e7a237c67a1b9ff92c04e89969d22d1 \ - --hash=sha256:708382875c3cfe91be625dddcba03dee2dfdadbad2c431568a8c7f2f2af0bbee \ - --hash=sha256:70e655c3aa5f893807830f549a7275031a181dbebeaf74c461b51adc755d9335 \ - --hash=sha256:735be62d757e1e7e496ccb6401efe82b473faa653e95eec0826cd7819a29a34c \ - --hash=sha256:739ee81e574f33b46f1e6d9312f3ec2c549bdd574a4ebb6bf106775c9d85ca7b \ - --hash=sha256:7469912c1a04102457871ff675aebe600dbb7e79a6450a166cc8079b88f6ca79 \ - --hash=sha256:75eefdf257a84ea73d553eb80d0abbff0af4c9df62529e4600fd3f96ff17eeb3 \ - --hash=sha256:76bd99f3508cb2ec87976f2e3fe7d92fb373a661cacffb863013d15e4cfaf0eb \ - --hash=sha256:78a55f320ca924545494ce153df02d4349156cd95dc4603c1f0e80c42c889249 \ - --hash=sha256:7ddffe7c0b3be414f88185e212758989c65b497315781290eb029e2c1e1fd64e \ - --hash=sha256:7fd0d0bd3a79e009dddbf5fedfd927ad495c218cd7b13a112d28a37e2079725c \ - --hash=sha256:7fe194f37052a8f45a1a8507e36229e28b79f3d21542ae55ea6a18c6a444f625 \ - --hash=sha256:82d5272d08451bcef6fb6235a0a04cf1816b6b6815cec76be5ace1de17e0c1a4 \ - --hash=sha256:830d181781bb263c9fa430b81f82c867546f5dcb368e73931c8591f533a04afb \ - --hash=sha256:88c6bea948da03087035bb9ca9625305d70e084aa33f11e17048cb7dda4ca293 \ - --hash=sha256:8a06f6c3d3b8d487226fdde93f621e04a54faecc5bf5d9b16497b8f9ead0ac3e \ - --hash=sha256:8dfe2f4840b043436ee5bdd07b0fa5daecedf086e6957e7df050a56ab6db078d \ - --hash=sha256:8ef11dd996208c5a96eab0683f7a17cb4b992948464d2498520efd75a10a2aac \ - --hash=sha256:91a06ee46b3e3ef3b237046b914c0c905eab9fdfeac677e9b51473b482e24c28 \ - --hash=sha256:972af65924ea25cf5b4d9326d549e69a9a4918d8a76a9d3a7cd174d98b237550 \ - --hash=sha256:a10d38fa300f6bef230fae1fdb4bc98706b78c8a3a2f3140fde748469ef3cfe8 \ - --hash=sha256:a1a083e9ee7a1904e545a6307c7db1dd76200077520fcbf7a98d886f81b57dd7 \ - --hash=sha256:a265e95e7087f44b88a6d78a63ea95a2ca0eb0a21ab4f76047f4c164a8beb413 \ - --hash=sha256:a404857e48d85a517bb5b974d740f8c4fccb25d8df98885f3a2a4d950870b845 \ - --hash=sha256:a4d2b27d8156922c9739dd2ff4f3934716e17cfd1cf6fb61aa17af7d378555e9 \ - --hash=sha256:ad0448ed5569f0a9b9b010af9fb5b6d9bdc0b4b877a3ddb188396c4742e62284 \ - --hash=sha256:b1a94e8935c69ae30043b465af496f447950f2c03660aee8657074084faae0b3 \ - --hash=sha256:b22d337c76b765cb7961d4ee47fe29f89e30921eb47bf856b14dc7641f4df3e5 \ - --hash=sha256:b809d1c861436a75b2b015dbfd94f6154fa4e7cb0a70e389df1d5f6246b21d1e \ - --hash=sha256:b960e700dc95faca7dd6919d0dce183ef89bfe01554d323cf5de7331a2e80f83 \ - --hash=sha256:bd37c087a538944652363cfd77fb7abe8100cc1f48afea0b88b38bf0f469c3d2 \ - --hash=sha256:c570649ceccfa5e11ad9351e9009dc484c315a51a56aa02ced07ae97644bb7aa \ - --hash=sha256:c89617a784e1c24a31f5aa4809402f8072a26b64ddbc437897f6391ff69b0ee9 \ - --hash=sha256:c93827e0854393cd904b927ceb529afc17776706f5b9e45c7eaf6a40b3fc7b25 \ - --hash=sha256:ca17ac727ffeeba6c46f5a88e0284a7cb1520fb03127645fcdd7041080adf849 \ - --hash=sha256:cc2085926872a1b26deda4b81b2254d2e5d2cb2c4d7b327abe4c820b7c93f40b \ - --hash=sha256:cc520f3f4af14f456143a534d554175dbc0f0636ffd653e63675cd591862a9d9 \ - --hash=sha256:d10c2ea7c90ba204cd053167ba214d0cdd00f3184c7b8d117a56d7fd2b0c6553 \ - --hash=sha256:d1b4426457757c30ad86b57cdbcc0adaa328399f1ec3d231a0a2ce7447248987 \ - --hash=sha256:d4d7ec24d3bfcfac3baf4dffd7fcea3fa8474b087ce32696232132064aa062cf \ - --hash=sha256:d774df483c12d6cba896eb9a1337bbc5ad3f564eb18cfaaee3e95fb4402f2a86 \ - --hash=sha256:d8930877a2dd40461968d9238d95c754e51b33ce7d2a45500f88ffeed5cb7202 \ - --hash=sha256:dd518f300e5eb6a8827bee380f2e1a31c01dc0af069b13abdecd4e5769bd8a97 \ - --hash=sha256:de1fff2683d8af01299ec01eb21a24b6097ce92015fc1fbefa977cecf076a3fc \ - --hash=sha256:de8d2c98777ba266f5466e211778d4e3bd00635a207c54f6f7511d8613b86dd3 \ - --hash=sha256:e0d450c9d444befe2668bf9386ac2945a2f38152248d58f6b3feea63db59ba08 \ - --hash=sha256:e478fb2a1391a56c8a6e10cc97c4a9002b410ecd1ac28c18d780661762e271bd \ - --hash=sha256:e89492dfcc4c27a718f8b5a4c8df1a2dec6c689718cccd70cb2ceba69ab8c642 \ - --hash=sha256:eab9ef8a9268042e819de03079b984eb0894f05a7b63c4e5319b1cf1ef362ba7 \ - --hash=sha256:ebae8f4a07372acfc3963fc8d68070cdaab70272c3dd836f057ebbe9b7d38643 \ - --hash=sha256:ec846bde66b75d68518c7b24a0a46d09db0aee5a6aefd2209d9901faf6e9df21 \ - --hash=sha256:f42e2c25d3eee4ea3da88466f38ed0dce8c622a1a9d92572e5ee53b7a6bb9ef1 \ - --hash=sha256:f556ea05d9c5f6dae50d57ce6234e4ab1fbf4551dd0d52b4fed6ef537d9f3d3c \ - --hash=sha256:f65d1b90e9ddc791ea82ef91a9ae0ab27ef6c0cfa88fadfa0e5ca5a22f8fa22f \ - --hash=sha256:fc43257a06e6117da6a8a0779243b974cdb9205fed82e32eb669f6746c75d27d \ - --hash=sha256:fd7ba56cf6340c732ecb78787c4e9600c4bd01372af7313ded21037126d33ec6 \ - --hash=sha256:ffd1a9f9babec9119712e76a39397d8aa0d72ef8c4ccad917c6175d7e7f81b74 \ - --hash=sha256:fff8584e3bbdc5c1713cd016fbf4b88babfffd4e5e89b39020f2a208dd24c900 - # via pyld -geomet==1.1.0 \ - --hash=sha256:4372fe4e286a34acc6f2e9308284850bd8c4aa5bc12065e2abbd4995900db12f \ - --hash=sha256:51e92231a0ef6aaa63ac20c443377ba78a303fd2ecd179dc3567de79f3c11605 - # via ckanext-dcat html5rdf==1.2.1 \ --hash=sha256:1f519121bc366af3e485310dc8041d2e86e5173c1a320fac3dc9d2604069b83e \ --hash=sha256:ace9b420ce52995bb4f05e7425eedf19e433c981dfe7a831ab391e2fa2e1a195 # via rdflib -lxml==6.1.3 \ - --hash=sha256:032a0a97eed428bd143c75a11118238546424ceb2fa311cca5f073aa44658dc4 \ - --hash=sha256:05f5bce9af14fd1506997594bd81cee6d9c6b58ea80a39c058327aa6371ed9e9 \ - --hash=sha256:0794e04ba343852c6d78e996c58ef4b8e579b4ecc72f8df0d4058bf843b4c96e \ - --hash=sha256:0ab2467e405e748d93495fb5568e74044802b8d3ff2b2a1607c3f78c6e982de5 \ - --hash=sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe \ - --hash=sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc \ - --hash=sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748 \ - --hash=sha256:0f17d83c48ee9dfd96abae3ac3e2108c76d2fc86ce96355e37b8da9f7f4ecc08 \ - --hash=sha256:0feebef8d0521188d0157f758356072e840173aa61ca45b8b3f87959ac283dd5 \ - --hash=sha256:13a620a3fcc20023f9e6ed5c383e00e826f1c2d5db554df2f67240760f9118e8 \ - --hash=sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741 \ - --hash=sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87 \ - --hash=sha256:16148acd77ed1d8836a56db883af2f5eed720f9723088110b16a0d08582130a6 \ - --hash=sha256:170773d8a3cdc76259065523ddd978c44f9806e28605f08812e8f86783e44ac6 \ - --hash=sha256:18293f8a8d8b6a8e71ef37706b659e3846a4261232158167b1ddf35f6994f633 \ - --hash=sha256:18a4db52b5a7b53a3540b0b0f4123319334621ee8083d496de314d0bf06ff59a \ - --hash=sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d \ - --hash=sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa \ - --hash=sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e \ - --hash=sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70 \ - --hash=sha256:20384c2bbcbf87180c8c61eb60869699c1ec0cd09b62cfd13804022d860b0867 \ - --hash=sha256:20428910dae17a1a93152a3ff2c0441d2f4932992c0797d65651dd0561f1792f \ - --hash=sha256:207dfc3d47cf0e575e643bbc140dacc8863b39abaa1e5307cd64c7f2365b8a12 \ - --hash=sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156 \ - --hash=sha256:2123e5aa075ac20d23c7af489255efd129cbfe190dbe88fd42598cc9df3199b6 \ - --hash=sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5 \ - --hash=sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75 \ - --hash=sha256:22eec57e26c418cde02c051ce9914a365e52a7f135a565c6f0480242aeebab48 \ - --hash=sha256:23c366231259cd75ad06495174701afb3fcb36a92917fa47de2d1f1bd9d95739 \ - --hash=sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37 \ - --hash=sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626 \ - --hash=sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015 \ - --hash=sha256:2b9b1325ca1c2a9a2dbb6eb913ae563313f2082ae60b03210f7e83ee80712274 \ - --hash=sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165 \ - --hash=sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e \ - --hash=sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79 \ - --hash=sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d \ - --hash=sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d \ - --hash=sha256:302f72413251c03f671e063c9414bed5dc8c927069e5abb69245521e51a4e81b \ - --hash=sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026 \ - --hash=sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad \ - --hash=sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11 \ - --hash=sha256:3847e71a78cbbc1aff955dbbbaf2fff12153f611d3162c5beaa3395636cbc2f9 \ - --hash=sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385 \ - --hash=sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7 \ - --hash=sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd \ - --hash=sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f \ - --hash=sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c \ - --hash=sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a \ - --hash=sha256:40bcbd9f94166ffe925811e730607385cec959f42fb1bb7dad83748680465221 \ - --hash=sha256:41096ec0740a58dad03d3ae0c7486d306d20becefb13ceb1649835ab3eb64167 \ - --hash=sha256:415e3a115c0d510e329020012834d1c0aa1c581ee53a218603e38abbc1dea70a \ - --hash=sha256:41e2d428110b408e963b6fb18f9bbf1f5c027b56bd4b498d54556476c0aeb1c3 \ - --hash=sha256:424aa5657141d306ba9ad1baab4b2c0a0719040075ee6c66aee9bb2dea2b5054 \ - --hash=sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245 \ - --hash=sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21 \ - --hash=sha256:4736e6c87e603146d8949d8501da621ad20c31015060d3fcf95ace2859f3e3e6 \ - --hash=sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e \ - --hash=sha256:49fbc2682a9306135b7ec49e93f97f9c26689b9b7f96ed2742d8d6497e994d13 \ - --hash=sha256:4a579dfb9c835f8ab47f4b8ed33440cbc75b806b73297208e6ec2a33e903740b \ - --hash=sha256:4b061064b4a2fe8598a466d723d43dbcd5a610a5d5cfe02fb6226f5c17349f75 \ - --hash=sha256:4e11e885e0704be185867fcf71b904d8f65d7d6877bc121f69870b0d0479ba7b \ - --hash=sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d \ - --hash=sha256:527195c188d7d0af748cd48d220ab8cdc5cb99be3d49ac4d9be7324d8abf9bc0 \ - --hash=sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69 \ - --hash=sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414 \ - --hash=sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d \ - --hash=sha256:5929d9df5e7e3379183be0e21f7d559618a5b61cb63280df6164019242e337ed \ - --hash=sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f \ - --hash=sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf \ - --hash=sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2 \ - --hash=sha256:5d12669a2c419b0e8dc423d23dea24bb82f6f9cb829f32e04674b0ba40322a7c \ - --hash=sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2 \ - --hash=sha256:61116cec57ed69aebc70f37a545eec095339bb829efbdabcfb97c51e9536e158 \ - --hash=sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d \ - --hash=sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d \ - --hash=sha256:6374e9e382e5a98c9c5e66d41b357b470da1c54bce30f17f9dc4bcc58436cc1c \ - --hash=sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861 \ - --hash=sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd \ - --hash=sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0 \ - --hash=sha256:6ba4fe5bfbef6811a8e49b3719cde373ad399006c0c1ac184b7297116ecbba5d \ - --hash=sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5 \ - --hash=sha256:6e1eb8a4cbffd5553680ad96be6680e364710656eced73d1dc90ec489df599a3 \ - --hash=sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0 \ - --hash=sha256:71532ebf30be0048a45559b4fab15333fbaaf9042f658e878d918ecd0cf09805 \ - --hash=sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a \ - --hash=sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8 \ - --hash=sha256:773062aec2f2e56b2b22d37054123f0de8a22a4688a0c3376c3fe42685f975cf \ - --hash=sha256:7ae4949f212a53b007dbc355884fda122545c5764a54256c9217e419a62a6559 \ - --hash=sha256:7b2bb7d703bed7ac893bf7f40d97b5d9279d35d2ce460624ca28929eab0d5a3d \ - --hash=sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c \ - --hash=sha256:7d5a748d12dd9b535e0a130f60dae9ddf0adafbabe61e7864f55c7436c84547a \ - --hash=sha256:7dd624c1eaa629ad44b59a1a0145fdf2d67895592dce94c9358b938b3d075e65 \ - --hash=sha256:7f75b9b9fec2a9c6b18095c81865580e795b1441c429e42d22fcc82a77f40039 \ - --hash=sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92 \ - --hash=sha256:8499d464de86fab0f102313cce32a9bed9ab1f06ec813cf025cb790964fbb765 \ - --hash=sha256:869dfcd4d381cb0ea87085cc4f011b9171b494ef21e76ad8665f6d5e2d1dc8a1 \ - --hash=sha256:8753b8d51dbc86fd335ee31fcf7f3658e9f5c016d4edfb23f76ad295f4b8c9d0 \ - --hash=sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1 \ - --hash=sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2 \ - --hash=sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758 \ - --hash=sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473 \ - --hash=sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310 \ - --hash=sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c \ - --hash=sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4 \ - --hash=sha256:92d96586376fb79a33474797186bf993250152ee5c32650b67db78d54b92e6f3 \ - --hash=sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17 \ - --hash=sha256:97acecb11cbc411473f15b8d780df06d7a9f3a2aad9aca78364f56640c8fb70e \ - --hash=sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9 \ - --hash=sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48 \ - --hash=sha256:9e67324961ac9bbe616cce5100514d2e34d88665aeb07071e8b16eac55d06d94 \ - --hash=sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a \ - --hash=sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2 \ - --hash=sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55 \ - --hash=sha256:a2e3f70673a1d5b82f38255f777d26cd855bf2092b1436c4867464a7892f9238 \ - --hash=sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e \ - --hash=sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56 \ - --hash=sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0 \ - --hash=sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0 \ - --hash=sha256:aa9fd1ee2a5dacfc41039ed49ffeeacfa75bafbd255b69f3b578e11897a0e623 \ - --hash=sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e \ - --hash=sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1 \ - --hash=sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a \ - --hash=sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c \ - --hash=sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed \ - --hash=sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6 \ - --hash=sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4 \ - --hash=sha256:b50343241eb69fd85f7791cf8bcc7b1c4729826b7d59ba2f6b27db29638fa745 \ - --hash=sha256:bc8dd3d9c93e70c3df974a201ac2958b6d77b465d813c51d1f15fa8e645763ae \ - --hash=sha256:be5346653c0b0e34be96869ff9dbeba23860156f89a2896a64c64fb419260cb6 \ - --hash=sha256:c00e26288784460885fe76e4d4b293573e0f791f52e6d60e27b42edf005922eb \ - --hash=sha256:c1b50797ac246bb2942a04b6c0f69af0667aba7cf7535f39bbb1b3208fd5d128 \ - --hash=sha256:c34ca1dc41bd86d9ff830d5bdf4e4a752bba6c54f7d2707027ce0eabd36084c9 \ - --hash=sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5 \ - --hash=sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9 \ - --hash=sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415 \ - --hash=sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8 \ - --hash=sha256:c66f858b82497173f73366795fc6ee8171620e75a338506d6b2e7bc16f5fca11 \ - --hash=sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8 \ - --hash=sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2 \ - --hash=sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a \ - --hash=sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300 \ - --hash=sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0 \ - --hash=sha256:cc669256d28736f7f3a149df5c380c50ace2692ba3e62203d10656fade4a2145 \ - --hash=sha256:ce1f220114959941170e22b8ad44279f6dee2dcef7591814d01ae805dc058889 \ - --hash=sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9 \ - --hash=sha256:d077f21f4b16f0471353883748f126f62038760397c107bb9fad2ca94dc0dfb7 \ - --hash=sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559 \ - --hash=sha256:d2765c18ce303149ee804b1f3dad11232726dd0a702d73a15cf19179ac8cc962 \ - --hash=sha256:d44442effeb8781f392340c5dc8c6716fba41dbeacb82fd4c0f09026fb5ff682 \ - --hash=sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e \ - --hash=sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb \ - --hash=sha256:d9a0d12846d6ce434fb3857918eef4315ec9b4769deb020c75828798614bfcfd \ - --hash=sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc \ - --hash=sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8 \ - --hash=sha256:da85db328e507da922d586c3c7416ec360ec22e9cd9e0700691afacde0c81f53 \ - --hash=sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e \ - --hash=sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed \ - --hash=sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d \ - --hash=sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32 \ - --hash=sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477 \ - --hash=sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023 \ - --hash=sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887 \ - --hash=sha256:ea2c01cdb16dc12156e455007c406dfaaece0c89aa4ba0e3b47586779f951d41 \ - --hash=sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6 \ - --hash=sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376 \ - --hash=sha256:ec295280f4b37769256da025acf5890370355ac589c27e89caae0b5e9eedc702 \ - --hash=sha256:f6449672f9c93316deb5e2839e18931f468670e44d5bd9b1301a5a9655d45c07 \ - --hash=sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5 \ - --hash=sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2 \ - --hash=sha256:f8b9c8ceebae6387d0dc77f7f4dbbfbfc962dba2efbfe6877486075a480726b4 \ - --hash=sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011 \ - --hash=sha256:fbfb70ba01355251faf6b293171df49f73a88a1b6494db109ffea85442574458 \ - --hash=sha256:fe91993149523aa59941b9e3c90e2eb45f57ad014697aef6c8b13339a59c019e \ - --hash=sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0 \ - --hash=sha256:ff88a92cafde90888511242d1c54afcc1a8adbb6dc0a88fa7f87e29e92400d4a - # via pyld owlrl==7.6.2 \ --hash=sha256:83347bf7f133979e87b2b18695d51d25510b99cec3f6919b5df05d4fbf058ae0 \ --hash=sha256:c743f35c2d908396e77823852bb1ebbce88340cd49961493983bec42c93283a8 @@ -306,10 +16,6 @@ prettytable==3.18.0 \ --hash=sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680 \ --hash=sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a # via pyshacl -pyld==3.3.0 \ - --hash=sha256:58aa63542a5989431426bdfef932db0d1b974e2928e179f74d5a903804de9692 \ - --hash=sha256:cf5353786582056e76cb1698db32d8fc796250703311d0e5e33dd3e86f23e031 - # via ckanext-dcat pyparsing==3.3.3 \ --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 @@ -323,13 +29,8 @@ rdflib==7.6.0 \ --hash=sha256:6c831288d5e4a5a7ece85d0ccde9877d512a3d0f02d7c06455d00d6d0ea379df # via # -r conformance/requirements.txt - # ckanext-dcat # owlrl # pyshacl -six==1.17.0 \ - --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ - --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 - # via ckantoolkit wcwidth==0.9.2 \ --hash=sha256:0a47e03d8293590ecce66c45dc20ff7b4b885e3c78093722239585eca0d77ab2 \ --hash=sha256:0cd4f7f2e53905dcb110d213a4c8529b6733fa3d232d8c717f946cc69a10349b \ diff --git a/conformance/requirements.txt b/conformance/requirements.txt index 8144ff7..3e0a137 100644 --- a/conformance/requirements.txt +++ b/conformance/requirements.txt @@ -2,9 +2,9 @@ # # These are the only Python in the project (everything shipped is Rust). They run # out-of-band in CI, isolated from the -# pure-Rust build, and are never shipped in either binary. The three TOP-LEVEL +# pure-Rust build, and are never shipped in either binary. The two TOP-LEVEL # versions below are pinned and the jobs install into a fresh venv, so a -# pyshacl/rdflib/ckanext-dcat bump is a deliberate, reviewed change. +# pyshacl/rdflib bump is a deliberate, reviewed change. # # This file is the human-facing, Dependabot-visible pin list. The venv is actually # installed from the fully-resolved, hash-locked `requirements.lock` (transitive deps @@ -20,25 +20,7 @@ # the file no longer depends on who regenerated it. # `ci-local.sh assert_lock_current` fails the gate if the lock does not track these pins. # -# - pyshacl / rdflib: the SHACL engine + RDF graph for check_fdp.py / check_dataset.py. -# - ckanext-dcat: the consumer-compat profile parser for check_ckanext.py. The -# userportal deploys the GDI *fork* installed from git -# (`gdi-userportal-ckanext-dcat`, with the harvester -# `gdi-userportal-ckanext-fairdatapoint`), at the tags named in -# gdi-userportal-ckan-docker's `ckan/Dockerfile`. We pin the fork's -# upstream base here, not the git fork: the fork's `RDFParser` imports -# `ckan.plugins`, so the real-parser tier cannot load in this CKAN-less -# venv either way — check_ckanext.py always degrades to its best-effort -# direct rdflib consumer-parse (it says so in its output), so a git -# install would only add per-run network fragility for zero functional -# gain. pyshacl is the must-have gate; ckanext is nice-to-have. -# PIN NOTE: check_ckanext.py's fallback hand-mirrors the predicate reads -# of the fork's `euro_health_dcat_ap` profile. `scripts/vendored.sh pins` -# watches that file (and the harvester's subclass) by hash at the deployed -# tag, and the fork's `pyproject.toml` version against the pin below. When -# it drifts: re-verify the mirrored list against the fork's -# `profiles/euro_health_dcat_ap.py` (and `euro_dcat_ap_3.py`), re-pin the -# hash, and bump the version below if the base moved. +# - pyshacl / rdflib: the SHACL engine + RDF graph for check_fdp.py, check_dataset.py and +# check_ckanext.py. pyshacl==0.40.1 rdflib==7.6.0 -ckanext-dcat==2.4.4 diff --git a/crates/gdi-node-standalone/tests/conformance_crawl.rs b/crates/gdi-node-standalone/tests/conformance_crawl.rs index ea8755c..9902488 100644 --- a/crates/gdi-node-standalone/tests/conformance_crawl.rs +++ b/crates/gdi-node-standalone/tests/conformance_crawl.rs @@ -12,7 +12,7 @@ //! It then invokes the Python conformance checks (`conformance/check_fdp.py`, //! `conformance/check_ckanext.py`) over that union via the conformance venv, //! asserting that pySHACL conforms on both shape sets, with only the `dct:hasPart` minCount-0 -//! relaxation, and that the consumer parser round-trips the graph. +//! relaxation, and that every dataset carries the fields the harvest profile reads. //! //! The test is `#[ignore]` because it needs the Python venv. Run it with `--ignored` and //! `GDI_FDP_VENV` or `GDI_FDP_PYTHON` pointing at the venv. See `conformance/README.md`. @@ -780,13 +780,13 @@ async fn fdp_union_conforms_to_shapes_and_consumer() { "negative SHACL meta-validation failed (the shapes did not reject a malformed graph):\n{neg_out}" ); - // ckanext-dcat is optional: it degrades gracefully, exiting 0 with a skip note if the - // standalone wiring fails. A success here means it either round-tripped or skipped. + // Also required: every dataset carries each field the harvest profile reads; several + // have no SHACL cover. let (ckan_ok, ckan_out) = run_check(&py, "check_ckanext.py", &union_path); println!("{ckan_out}"); assert!( ckan_ok, - "ckanext-dcat check returned a hard failure (it should degrade gracefully):\n{ckan_out}" + "harvest-profile consumer check failed:\n{ckan_out}" ); } diff --git a/docs/testing.md b/docs/testing.md index 99946e3..a316ebf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -128,10 +128,11 @@ Standards conformance is guarded on two sides: asserts the exact ignored-test count per target. `beacon_schema_conformance.rs` is kept separate by convention; it carries no `#[ignore]`d tests and runs in the ordinary `cargo test`/nextest sweep. -- **Python** (in `conformance/`, isolated from the Rust build, never shipped): pySHACL + - `ckanext-dcat` over the crawled FDP graph (`check_fdp.py`, `check_fdp_negative.py`, - `check_ckanext.py`, `check_dataset.py`), and a Rust↔Python `crypt4gh` interop round-trip - (`crates/core/tests/crypt4gh_interop.rs` against pinned `crypt4gh==1.8.6`). +- **Python** (in `conformance/`, isolated from the Rust build, never shipped): pySHACL and + a harvest-profile read check over the crawled FDP graph (`check_fdp.py`, + `check_fdp_negative.py`, `check_ckanext.py`, `check_dataset.py`), and a Rust↔Python + `crypt4gh` interop round-trip (`crates/core/tests/crypt4gh_interop.rs` against pinned + `crypt4gh==1.8.6`). The Python-dependent Rust tests are `#[ignore]` by default (they need a venv discovered via `GDI_FDP_PYTHON`/`GDI_FDP_VENV` / `C4GH_VENV`), so a plain `cargo test` needs no diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 053198d..2bdcb2b 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -136,10 +136,10 @@ # pip-audit advisory scan of the conformance and crypt4gh Python lockfiles # sbom cargo-cyclonedx SBOM generation # pins (network) the upstream content this tree depends on still is what it -# was pinned against: the userportal's deployed profile code, the -# vendored gdi-metadata shapes, the beacon-v2 release, the GitHub action -# pins (`uses: owner/repo@sha # tag`) and the Dockerfile base-image -# digest (needs Docker; skips visibly without it). Drift and an +# was pinned against: the vendored gdi-metadata shapes against upstream, +# the beacon-v2 release, the GitHub action pins +# (`uses: owner/repo@sha # tag`) and the Dockerfile base-image digest +# (needs Docker; skips visibly without it). Drift and an # unreachable host are both warnings here: neither is a defect in this # tree. # pins-strict the same check with drift fatal, which is what `release` runs @@ -245,8 +245,8 @@ # GATE_LOCK the slot's lock file (default: # /tmp/gdi-node-standalone-gate-.lock). One file per machine, # shared by every checkout, because what they contend for is the CPU. -# PINS_STRICT set to 1 to make pin drift (userportal profile content, gdi-metadata -# shapes, external and action pins) fatal instead of a warning. Set by +# PINS_STRICT set to 1 to make pin drift (gdi-metadata shapes, external and action +# pins) fatal instead of a warning. Set by # the weekly `pins` job, `pins-strict` and `release`. # BASE_IMAGE_STRICT set to 1 to make a rebuilt base image fatal too; otherwise it is a # note. Set by `pins-strict` and `release`, not by the weekly job. @@ -634,7 +634,7 @@ machete() { } # The Python half of the advisory scan. `cargo deny` covers every Rust crate and cannot see -# Python, so the conformance and crypt4gh closures (pyshacl, rdflib, lxml, and the +# Python, so the conformance and crypt4gh closures (pyshacl, rdflib, and the # cryptography/PyNaCl stack behind crypt4gh) would otherwise have no advisory gate. The # hash-locked lockfiles make that closure reproducible; they do not make it audited, and a # pinned-and-hashed vulnerable version is exactly as vulnerable. @@ -1304,7 +1304,7 @@ _base_image_digest_check() { # _base_image_digest_check # `sed -n '…/p'` prints every matching line. `head -n1` makes "first line wins" the # contract rather than an accident of today's output. Kept separate from # `_base_image_freshness` below so it is unit-testable without Docker: feed it canned -# multi-`Digest:` text on stdin. +# inspect output on stdin. _first_upstream_digest() { sed -n 's/^Digest: *\(sha256:[0-9a-f]*\).*/\1/p' | head -n1 } @@ -1375,20 +1375,19 @@ _base_image_pin_gate() { # without wifi into a red gate. `vendored.sh pins` returns 1 for real drift and 2 for # unreachable, and only the former is fatal here. An unreachable check is reported loudly # and skipped: it is a verdict about the network, not about the pins. Drift is a verdict -# about the federation rather than about this tree, so it does not fail `all` either. +# about upstream rather than about this tree, so it does not fail `all` either. # # Everything this leg watches lives in repositories this project does not control: the -# userportal's deployed profile code, the gdi-metadata shapes, the beacon-v2 release tag, -# the action tags. Any of them can move because someone else committed, and `all_legs()` -# is a bare loop under `set -e`, so a fatal verdict here would take every later leg with -# it. Upstream moving is not a defect in this tree, and a gate that reddens for it is a -# gate people learn to ignore. +# gdi-metadata shapes, the beacon-v2 release tag, the action tags. Any of them can move +# because someone else committed, and `all_legs()` is a bare loop under `set -e`, so a fatal +# verdict here would take every later leg with it. Upstream moving is not a defect in this +# tree, and a gate that reddens for it is a gate people learn to ignore. # # The signal is relocated rather than weakened: drift is reported on every run, and is # fatal where being behind costs something: the weekly `pins` job and `pins-strict` # (PINS_STRICT=1), and `release` before any artifact ships. pins() { - step "pins — userportal profile content, gdi-metadata shapes, release + action pins (network); base image digest (Docker)" + step "pins — gdi-metadata shapes, release + action pins (network); base image digest (Docker)" # Every half reports before any verdict is acted on. Strict-mode drift is collected # and fails the leg once, at the end, naming every half that drifted. local fatal=() rc=0 brc=0 @@ -1404,9 +1403,9 @@ pins() { 2) printf '%sWARNING: external pins could not be fetched (offline / upstream down) — SKIPPED, not verified. Re-run scripts/ci-local.sh pins when connected.%s\n' "$C_ERR" "$C_OFF" ;; *) if [[ "${PINS_STRICT:-0}" == "1" ]]; then - fatal+=("external pin drift — see above. Follow the federation in lockstep (re-check what the drifted line names, then re-pin it in scripts/vendored.sh); do not get ahead of it.") + fatal+=("external pin drift — see above. Follow upstream in lockstep (re-check what the drifted line names, then re-pin it in scripts/vendored.sh); do not get ahead of it.") else - printf '%sWARNING: external pin DRIFT — see above. The federation moved; this tree did not break.\n' "$C_ERR" + printf '%sWARNING: external pin DRIFT — see above. Upstream moved; this tree did not break.\n' "$C_ERR" printf ' Follow it in lockstep (re-check what the drifted line names, then re-pin it in\n' printf ' scripts/vendored.sh); do not get ahead of it. Not fatal here — `release` and\n' printf ' `pins-strict` fail closed on this.%s\n' "$C_OFF" @@ -2043,10 +2042,8 @@ release() { } # The full vendored-integrity check: re-fetches every vendored file from its pinned -# upstream commit, asserts byte-identity, then checks the userportal profile content and the -# external pins. `vendored_files`, -# in `all`, only counts what is on disk; this is what proves the content still matches what -# was reviewed. +# upstream commit and asserts byte-identity. `vendored_files`, in `all`, only counts what is +# on disk; this is what proves the content still matches what was reviewed. # # It needs the network, so it stays out of `all` and runs in `release`, before any artifact # ships. @@ -2235,7 +2232,7 @@ ensure_venv() { # ensure_venv step "venv — creating $dir from $lock (hash-locked; one-time; needs network)" # Build this venv on the pinned interpreter, unlike the stdlib-only guard legs, which are # version-agnostic and run on ambient `python3`. The difference is what goes into it: - # pySHACL, rdflib, lxml and cryptography are third-party, their wheels are built per + # pySHACL, rdflib and cryptography are third-party, cryptography's wheels are built per # interpreter, and these venvs produce the conformance and crypt4gh verdicts. The workflow # jobs build them under `actions/setup-python` at .python-version, so a local venv on some # other interpreter renders its verdict on a different closure. diff --git a/scripts/tests/test_action_pin_comments.py b/scripts/tests/test_action_pin_comments.py index 1971e20..bfd5e23 100644 --- a/scripts/tests/test_action_pin_comments.py +++ b/scripts/tests/test_action_pin_comments.py @@ -1,4 +1,4 @@ -"""Guard: every SHA-pinned `uses:` carries a version comment, and `pins` verifies them. +"""Guard: every SHA-pinned `uses:` carries the version comment `pins` verifies. The network half lives in `scripts/vendored.sh check_action_pins`, which compares each `uses: owner/repo@ # ` against what upstream says that tag resolves to. That @@ -15,7 +15,7 @@ import re import unittest -from _helpers import REPO_ROOT, strip_comments +from _helpers import REPO_ROOT WORKFLOWS = REPO_ROOT / ".github" / "workflows" @@ -79,29 +79,6 @@ def test_no_action_is_pinned_to_a_mutable_ref(self): "us. Pin the 40-hex commit SHA and put the version in a trailing comment.", ) - def test_the_pins_leg_actually_runs_the_action_check(self): - """Wiring guard: the check exists and the `pins` command calls it. - - A `check_action_pins` that is defined but never invoked is a guard that verifies - nothing. - """ - # Read what the script runs, not what its comments say: a comment restating the - # call would otherwise satisfy this assertion. - script = strip_comments( - (REPO_ROOT / "scripts" / "vendored.sh").read_text(encoding="utf-8") - ) - self.assertIn( - "check_action_pins()", - script, - "check_action_pins is not defined in vendored.sh", - ) - # The dispatch must invoke it, not merely define it. - self.assertRegex( - script, - r"arc=0;\s*check_action_pins", - "the `pins` dispatch does not invoke check_action_pins", - ) - if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_base_image_digest_pin.py b/scripts/tests/test_base_image_digest_pin.py index 6089127..17394fa 100755 --- a/scripts/tests/test_base_image_digest_pin.py +++ b/scripts/tests/test_base_image_digest_pin.py @@ -11,10 +11,7 @@ extracts that function's source out of ci-local.sh and runs it with equal and unequal digest pairs. -It also covers `_first_upstream_digest`, the parser for that inspect output. `sed -n -'…/p'` prints every matching line, so a rendering carrying more than one `Digest:` line -would yield a multi-line "digest" that could only ever compare as drift against the -single-line pin. A synthetic multi-`Digest:` fixture pins the single-line result. +It also covers `_first_upstream_digest`, the parser for that inspect output. """ import re @@ -120,13 +117,6 @@ def test_unequal_digests_report_drift_and_fail(self): self.assertIn("upstream", proc.stdout) self.assertNotIn("ok:", proc.stdout) - def test_same_object_different_case_is_not_silently_equal(self): - # The comparison must be a literal string match, not a case-normalised one: - # sha256 digests are lowercase by convention, and two digests that differ only in - # case are still two different digests. - proc = run_check(DIGEST_A, DIGEST_A.upper()) - self.assertEqual(1, proc.returncode, proc.stderr) - def run_first_upstream_digest(stdin_text: str) -> subprocess.CompletedProcess: """Pipe `stdin_text` through `_first_upstream_digest` in isolation.""" @@ -143,23 +133,6 @@ def run_first_upstream_digest(stdin_text: str) -> subprocess.CompletedProcess: class FirstUpstreamDigest(unittest.TestCase): - def test_a_single_digest_line_is_returned(self): - proc = run_first_upstream_digest( - f"Name: whatever\nDigest: {DIGEST_A}\n" - ) - self.assertEqual(0, proc.returncode, proc.stderr) - self.assertEqual(DIGEST_A, proc.stdout.strip()) - - def test_only_the_first_of_two_digest_lines_is_returned(self): - # A rendering with a second `Digest:` line must not turn the result into a - # two-line string. - proc = run_first_upstream_digest( - f"Digest: {DIGEST_A}\nDigest: {DIGEST_B}\n" - ) - self.assertEqual(0, proc.returncode, proc.stderr) - self.assertEqual(DIGEST_A, proc.stdout.strip()) - self.assertNotIn(DIGEST_B, proc.stdout) - def test_realistic_multi_platform_output_yields_only_the_manifest_list_digest(self): # The shape `docker buildx imagetools inspect` prints for a multi-arch image: one # `Digest:` line for the manifest list, then each platform's own digest inside a diff --git a/scripts/tests/test_gate_ordering.py b/scripts/tests/test_gate_ordering.py index 9d377d8..9673e84 100755 --- a/scripts/tests/test_gate_ordering.py +++ b/scripts/tests/test_gate_ordering.py @@ -428,22 +428,7 @@ def run_pins( class PinDriftFatalityTest(unittest.TestCase): - """`pins()` executed: drift is fatal only under PINS_STRICT=1, and both halves report. - - The rule is about control flow, so the function is run rather than matched as text. A - regex over its source once required the strict `die` to sit inside the leg, the shape - that let a base-image drift end the leg before the external pins were fetched. - """ - - def test_strict_base_drift_does_not_hide_the_external_pins(self): - proc = run_pins(base_drift=True, external_rc=0, strict=True, base_strict=True) - self.assertIn( - "stub: vendored.sh pins ran", - proc.stdout, - "a strict base-image drift ended the leg before the external pins ran", - ) - self.assertEqual(1, proc.returncode, proc.stderr) - self.assertIn("base image digest drift", proc.stderr) + """`pins()` executed: drift is fatal only under PINS_STRICT=1, and every half reports.""" def test_strict_drift_in_both_halves_names_both(self): proc = run_pins(base_drift=True, external_rc=1, strict=True, base_strict=True) @@ -457,14 +442,6 @@ def test_strict_external_drift_alone_is_fatal(self): self.assertIn("external pin drift", proc.stderr) self.assertNotIn("base image digest drift", proc.stderr) - def test_strict_base_drift_survives_unreachable_external_pins(self): - # Unreachable is not fatal on its own, so its arm must not end the leg with a - # success that swallows the base-image verdict held above it. - proc = run_pins(base_drift=True, external_rc=2, strict=True, base_strict=True) - self.assertIn("could not be fetched", proc.stdout) - self.assertEqual(1, proc.returncode, proc.stderr) - self.assertIn("base image digest drift", proc.stderr) - def test_unreachable_external_pins_are_not_fatal_under_strict_mode(self): proc = run_pins(base_drift=False, external_rc=2, strict=True) self.assertEqual(0, proc.returncode, proc.stderr) @@ -519,17 +496,6 @@ def test_inconsistent_dockerfile_pins_are_fatal_in_every_mode(self): self.assertIn("different digests", proc.stderr.lower()) -class PinsStrictTargetTest(unittest.TestCase): - def test_pins_strict_makes_both_halves_fatal(self): - text = CI_LOCAL.read_text(encoding="utf-8") - self.assertRegex( - text, - r"(?m)^\s*pins-strict\)\s+(?=.*\bPINS_STRICT=1\b)(?=.*\bBASE_IMAGE_STRICT=1\b).*\bpins\b", - "the `pins-strict` target must set PINS_STRICT=1 and BASE_IMAGE_STRICT=1, since " - "it is documented as the check `release` runs", - ) - - class ShapeDriftFatalityTest(unittest.TestCase): """The gdi-metadata shapes half of `pins()`: `vendored.sh drift gdi-metadata`. @@ -558,17 +524,6 @@ def test_a_set_filter_that_matches_nothing_is_fatal_in_every_mode(self): proc = run_pins(base_drift=False, external_rc=0, strict=strict, shapes_rc=1) self.assertEqual(1, proc.returncode, (strict, proc.stdout, proc.stderr)) - def test_an_unexpected_drift_exit_still_names_the_other_findings(self): - # The leg promises one failure naming every half; a solo `die` here would eat the - # base-image and external verdicts collected above it. - proc = run_pins( - base_drift=True, external_rc=1, strict=True, base_strict=True, shapes_rc=1 - ) - self.assertEqual(1, proc.returncode, proc.stderr) - self.assertIn("base image digest drift", proc.stderr) - self.assertIn("external pin drift", proc.stderr) - self.assertIn("drift gdi-metadata exited 1", proc.stderr) - class AnchoringTest(unittest.TestCase): def test_dev_reset_anchors_before_it_reads_the_working_directory(self): diff --git a/scripts/tests/test_userportal_profile_pins.py b/scripts/tests/test_userportal_profile_pins.py deleted file mode 100755 index b24807b..0000000 --- a/scripts/tests/test_userportal_profile_pins.py +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env python3 -"""Guard: `vendored.sh pins` watches what the userportal's profile reads, not which tag it deploys. - -`check_ckanext.py` hand-mirrors the deployed harvest profile, and the conformance venv pins -the GDI dcat fork's upstream base. Both are facts about the forks at whatever tag the -userportal deploys, so `check_userportal_profiles` reads the tags from the userportal's own -`ckan/Dockerfile` and compares content at those tags: a `sha256=` entry by hash, a -`requirement=` entry against the version `conformance/requirements.txt` pins. - -Run against a stub `curl` on PATH that serves fixture files by URL. Exit codes follow -`check_pins`: 0 clean, 1 drift, 2 unreachable. -""" - -import hashlib -import os -import re -import stat -import subprocess -import tempfile -import unittest -from pathlib import Path - -from _helpers import SCRIPTS, strip_comments - -VENDORED = SCRIPTS / "vendored.sh" -RAW = "https://raw.githubusercontent.com" - - -def _extract(name: str) -> str: - src = VENDORED.read_text(encoding="utf-8") - m = re.search(rf"^{re.escape(name)}\(\) \{{.*?^\}}", src, re.MULTILINE | re.DOTALL) - assert m, f"{name} is no longer defined in vendored.sh in an extractable form" - return m.group(0) - - -#: Serves `$STUB_FILES/`, 404 (curl exit 22) for anything else, and -#: curl exit 7 with no HTTP code when STUB_NET_DOWN=1. -STUB_CURL = """#!/usr/bin/env bash -url=""; out="" -while [ $# -gt 0 ]; do - case "$1" in - -o) out="$2"; shift ;; - https://*) url="$1" ;; - esac - shift -done -if [ "${STUB_NET_DOWN:-0}" = 1 ]; then printf '000 '; exit 7; fi -key="${url#https://raw.githubusercontent.com/}"; key="${key//\\//__}" -if [ -f "$STUB_FILES/$key" ]; then cp "$STUB_FILES/$key" "$out"; printf '200 41'; exit 0; fi -printf '404 41'; exit 22 -""" - -HEALTH_PROFILE = "class EuropeanHealthDCATAPProfile:\n reads = ['hdab']\n" -FDP_PROFILE = ( - "class FAIRDataPointDCATAPProfile(EuropeanHealthDCATAPProfile):\n pass\n" -) - - -def sha256(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - - -def dockerfile( - dcat_ref: str = "v9.9.9", - fdp_ref: str = "v1.0.0", - fdp: bool = True, - quoted: bool = False, -) -> str: - # The userportal installs forks in two shapes: `-e git+URL.git@ref#egg=name`, and the - # quoted `'name[extras] @ git+URL.git@ref'` it already uses for ckanext-scheming. - dcat = ( - f" 'ckanext-dcat[x] @ git+https://github.com/Org/fork-dcat.git@{dcat_ref}' \\" - if quoted - else f" -e git+https://github.com/Org/fork-dcat.git@{dcat_ref}#egg=ckanext-dcat \\" - ) - lines = ["RUN pip install \\", dcat] - if fdp: - lines.append( - f" -e git+https://github.com/Org/fork-fdp.git@{fdp_ref}#egg=ckanext-fairdatapoint \\" - ) - lines.append(" ckanext-dcat") - return "\n".join(lines) + "\n" - - -def pyproject(version: str) -> str: - # A dependency string carrying the pinned version too, so an unanchored match on the - # bare number would pass a moved [project] version. - return ( - "[project]\n" - 'name = "ckanext-dcat"\n' - f'version = "{version}"\n' - 'dependencies = ["ckanext-scheming>=2.4.4"]\n' - ) - - -class UserportalProfilePinsTest(unittest.TestCase): - def setUp(self): - self.tmp = Path(tempfile.mkdtemp(prefix="gdi-userportal-pins-")) - self.addCleanup( - lambda: subprocess.run(["rm", "-rf", str(self.tmp)], check=False) - ) - stub_dir = self.tmp / "bin" - stub_dir.mkdir() - stub = stub_dir / "curl" - stub.write_text(STUB_CURL, encoding="utf-8") - stub.chmod(stub.stat().st_mode | stat.S_IXUSR) - self.files = self.tmp / "files" - self.files.mkdir() - (self.tmp / "conformance").mkdir() - (self.tmp / "conformance" / "requirements.txt").write_text( - "pyshacl==0.40.1\nckanext-dcat==2.4.4\nrdflib==7.6.0\n", encoding="utf-8" - ) - self.serve("Org/portal/main/ckan/Dockerfile", dockerfile()) - self.serve("Org/fork-dcat/v9.9.9/pyproject.toml", pyproject("2.4.4")) - self.serve( - "Org/fork-dcat/v9.9.9/ckanext/dcat/profiles/health.py", HEALTH_PROFILE - ) - self.serve("Org/fork-fdp/v1.0.0/ckanext/fairdatapoint/profiles.py", FDP_PROFILE) - - def serve(self, path: str, content: str) -> None: - (self.files / path.replace("/", "__")).write_text(content, encoding="utf-8") - - def run_check(self, net_down: bool = False) -> subprocess.CompletedProcess: - pins = [ - "Org/fork-dcat|pyproject.toml|requirement=ckanext-dcat|dcat base", - f"Org/fork-dcat|ckanext/dcat/profiles/health.py|sha256={sha256(HEALTH_PROFILE)}|health profile", - f"Org/fork-fdp|ckanext/fairdatapoint/profiles.py|sha256={sha256(FDP_PROFILE)}|fdp profile", - ] - script = "\n".join( - [ - "set -euo pipefail", - f"RAW={RAW!r}", - 'USERPORTAL_DOCKERFILE="Org/portal/main|ckan/Dockerfile"', - "USERPORTAL_PROFILE_PINS=(", - *(f" {p!r}" for p in pins), - ")", - _extract("curl_to"), - _extract("classify_fetch_failure"), - _extract("check_userportal_profiles"), - "rc=0; check_userportal_profiles || rc=$?", - 'echo "rc=$rc"', - ] - ) - env = { - **os.environ, - "PATH": f"{self.tmp / 'bin'}{os.pathsep}{os.environ['PATH']}", - "STUB_FILES": str(self.files), - "STUB_NET_DOWN": "1" if net_down else "0", - "GITHUB_TOKEN": "", - } - return subprocess.run( - ["bash", "-c", script], - cwd=self.tmp, - env=env, - capture_output=True, - text=True, - check=False, - ) - - def rc(self, proc: subprocess.CompletedProcess) -> int: - m = re.search(r"^rc=(\d+)$", proc.stdout, re.MULTILINE) - self.assertIsNotNone(m, f"no rc line:\n{proc.stdout}\n{proc.stderr}") - return int(m.group(1)) - - def test_matching_content_at_the_deployed_tags_passes(self): - proc = self.run_check() - self.assertEqual(0, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex( - proc.stdout, r"OK\s+health profile\s+\(Org/fork-dcat@v9\.9\.9\)" - ) - self.assertRegex(proc.stdout, r"OK\s+fdp profile\s+\(Org/fork-fdp@v1\.0\.0\)") - self.assertRegex(proc.stdout, r"OK\s+dcat base\s+\(Org/fork-dcat@v9\.9\.9") - - def test_a_new_tag_carrying_identical_content_is_not_drift(self): - # The reason this check exists: a userportal release that moves the tag but not - # the profile or the base must pass without anyone touching this tree. - self.serve("Org/portal/main/ckan/Dockerfile", dockerfile("v10.0.0", "v1.1.0")) - self.serve("Org/fork-dcat/v10.0.0/pyproject.toml", pyproject("2.4.4")) - self.serve( - "Org/fork-dcat/v10.0.0/ckanext/dcat/profiles/health.py", HEALTH_PROFILE - ) - self.serve("Org/fork-fdp/v1.1.0/ckanext/fairdatapoint/profiles.py", FDP_PROFILE) - proc = self.run_check() - self.assertEqual(0, self.rc(proc), proc.stdout + proc.stderr) - self.assertIn("@v10.0.0", proc.stdout) - - def test_the_quoted_install_form_resolves_the_tag_without_its_quote(self): - self.serve("Org/portal/main/ckan/Dockerfile", dockerfile(quoted=True)) - proc = self.run_check() - self.assertEqual(0, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex( - proc.stdout, r"OK\s+health profile\s+\(Org/fork-dcat@v9\.9\.9\)" - ) - - def test_changed_profile_content_at_the_deployed_tag_is_drift(self): - self.serve( - "Org/fork-dcat/v9.9.9/ckanext/dcat/profiles/health.py", - HEALTH_PROFILE + " reads.append('extra')\n", - ) - proc = self.run_check() - self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex( - proc.stdout, r"DRIFT\s+health profile\s+\(Org/fork-dcat@v9\.9\.9" - ) - self.assertRegex(proc.stdout, r"OK\s+fdp profile") - - def test_a_moved_upstream_base_version_is_drift(self): - # The expected version is the one in requirements.txt; nothing else states it. - self.serve("Org/fork-dcat/v9.9.9/pyproject.toml", pyproject("2.5.0")) - proc = self.run_check() - self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex(proc.stdout, r"DRIFT\s+dcat base\s+\(Org/fork-dcat@v9\.9\.9") - self.assertIn("2.5.0", proc.stdout) - self.assertIn("2.4.4", proc.stdout) - - def test_a_fork_no_longer_installed_from_the_dockerfile_is_drift(self): - self.serve("Org/portal/main/ckan/Dockerfile", dockerfile(fdp=False)) - proc = self.run_check() - self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex(proc.stdout, r"GONE\s+fdp profile\s+\(Org/fork-fdp") - self.assertRegex(proc.stdout, r"OK\s+health profile") - - def test_a_pinned_path_missing_at_the_deployed_tag_is_drift(self): - ( - self.files / "Org__fork-fdp__v1.0.0__ckanext__fairdatapoint__profiles.py" - ).unlink() - proc = self.run_check() - self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex(proc.stdout, r"GONE\s+fdp profile\s+\(HTTP 404") - - def test_a_missing_dockerfile_is_drift_not_unreachable(self): - # The userportal moving its Dockerfile is the deployment changing shape; read as - # a network failure, every later run would pass with a warning. - (self.files / "Org__portal__main__ckan__Dockerfile").unlink() - proc = self.run_check() - self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) - self.assertRegex(proc.stdout, r"GONE\s+userportal ckan/Dockerfile") - - def test_no_network_is_unreachable_not_drift(self): - proc = self.run_check(net_down=True) - self.assertEqual(2, self.rc(proc), proc.stdout + proc.stderr) - self.assertIn("UNREACHABLE", proc.stdout) - self.assertNotIn("DRIFT", proc.stdout) - - -class ShippedPinsTest(unittest.TestCase): - def test_the_shipped_pins_are_well_formed(self): - src = VENDORED.read_text(encoding="utf-8") - m = re.search( - r"^USERPORTAL_PROFILE_PINS=\((.*?)^\)", src, re.MULTILINE | re.DOTALL - ) - assert m, "USERPORTAL_PROFILE_PINS is no longer an array literal in vendored.sh" - entries = re.findall(r'^\s*"([^"]+)"', m.group(1), re.MULTILINE) - reqs = (SCRIPTS.parent / "conformance" / "requirements.txt").read_text( - encoding="utf-8" - ) - self.assertTrue(entries) - for e in entries: - fields = e.split("|") - self.assertEqual(4, len(fields), e) - exp = fields[2] - self.assertRegex( - exp, r"^(sha256=[0-9a-f]{64}|requirement=[A-Za-z0-9_.-]+)$", e - ) - if exp.startswith("requirement="): - name = exp.split("=", 1)[1] - self.assertRegex( - reqs, - rf"(?m)^{re.escape(name)}==", - f"{name} is not pinned in conformance/requirements.txt", - ) - - def test_both_pin_subcommands_run_the_profile_check(self): - # Read what the script runs, not what its comments say. - src = strip_comments(VENDORED.read_text(encoding="utf-8")) - dispatch = src[src.index('case "$cmd" in') :] - for arm in ("check)", "pins)"): - body = dispatch[dispatch.index(arm) :] - body = body[: body.index(";;")] - self.assertIn("check_userportal_profiles", body, f"the `{arm}` arm") - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/tests/test_vendored_drift_filter.py b/scripts/tests/test_vendored_drift_filter.py index e7aea01..bd534a5 100755 --- a/scripts/tests/test_vendored_drift_filter.py +++ b/scripts/tests/test_vendored_drift_filter.py @@ -136,61 +136,6 @@ def test_a_shape_removed_upstream_is_drift_not_a_fetch_failure(self): self.assertRegex(proc.stdout, r"GONE\s+Resource\.ttl") self.assertNotIn("verdict about the", proc.stderr) - def run_drift_in_copy(self, mutate) -> subprocess.CompletedProcess: - """Run `drift gdi-metadata` from a copy of the tree with `mutate(set_dir)` applied. - - The script anchors on its own location, so a copy holding `scripts/vendored.sh` and - the one set is a complete tree for this subcommand. Upstream still serves the real - set, so whatever `mutate` did is a local fact, not drift. - """ - tree = self.tmp / "tree" - (tree / "scripts").mkdir(parents=True) - shutil.copy(VENDORED, tree / "scripts" / "vendored.sh") - copy = tree / "conformance" / "shapes" / "gdi-metadata" - shutil.copytree(SET_DIR, copy) - mutate(copy) - env = { - **os.environ, - "PATH": f"{self.tmp}{os.pathsep}{os.environ['PATH']}", - "STUB_URLS": str(self.urls), - "STUB_PREFIX": self.prefix, - "STUB_LOCAL": str(SET_DIR), - "STUB_DRIFT_FILE": "", - "STUB_NET_DOWN": "0", - "GITHUB_TOKEN": "", - } - return subprocess.run( - ["bash", str(tree / "scripts" / "vendored.sh"), "drift", "gdi-metadata"], - cwd=tree, - env=env, - capture_output=True, - text=True, - check=False, - ) - - def test_a_vendored_file_missing_locally_is_a_tree_defect_not_ok(self): - # Every surviving file matches upstream, so the counters alone say "clean"; the - # VENDORED.md file count is what knows a file is missing. - proc = self.run_drift_in_copy(lambda d: (d / "Resource.ttl").unlink()) - self.assertEqual(1, proc.returncode, proc.stdout + proc.stderr) - self.assertNotIn("OK: every vendored file", proc.stdout) - self.assertIn("FILE-COUNT MISMATCH", proc.stderr) - - def test_an_unparseable_vendored_md_exits_1_cleanly(self): - def blank_commit(d): - md = d / "VENDORED.md" - md.write_text( - re.sub( - r"(?m)^- \*\*Commit:\*\*.*$", "", md.read_text(encoding="utf-8") - ), - encoding="utf-8", - ) - - proc = self.run_drift_in_copy(blank_commit) - self.assertEqual(1, proc.returncode, proc.stdout + proc.stderr) - self.assertNotIn("unbound variable", proc.stderr) - self.assertIn("could not parse", proc.stderr) - if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_vendored_pins.py b/scripts/tests/test_vendored_pins.py index db5a31d..3445d77 100755 --- a/scripts/tests/test_vendored_pins.py +++ b/scripts/tests/test_vendored_pins.py @@ -46,7 +46,6 @@ def _extract(name: str) -> str: ok) printf '200 41'; exit 0 ;; ratelimited) printf '403 0'; exit 22 ;; forbidden) printf '403 41'; exit 22 ;; - old-curl) printf '403 %%header{x-ratelimit-remaining}'; exit 22 ;; esac """ @@ -93,13 +92,6 @@ def test_a_spent_budget_is_rate_limited_and_any_other_403_is_not(self): out, _ = self._curl_to("https://api.github.com/repos/x/y/commits/v1", "ok") self.assertEqual(out, "rc=0 code=200 ratelimited=0") - def test_an_old_curl_that_cannot_read_headers_never_claims_rate_limiting(self): - # curl < 7.83 prints `%header{...}` literally; that must read as unknown, not 0. - out, _ = self._curl_to( - "https://api.github.com/repos/x/y/commits/v1", "old-curl" - ) - self.assertEqual(out, "rc=22 code=403 ratelimited=0") - def test_the_token_goes_to_the_api_host_only(self): _, argv = self._curl_to( "https://api.github.com/repos/x/y/releases/latest", "ok", token="tok-1" @@ -119,23 +111,16 @@ def test_the_token_goes_to_the_api_host_only(self): class FetchFailuresAreClassifiedOnceTest(unittest.TestCase): """One classifier decides gone vs rate-limited vs unreachable, and every fetch site uses it. - Three hand-copied versions of the split let the fourth site omit it, which is how a - 404 on a vendored shape or on the userportal's Dockerfile read as "network" and passed - the strict leg. The classifier is where the flag `curl_to` sets becomes a line naming - the cause and, for a spent budget, GITHUB_TOKEN. + A site with its own copy of the split is how a 404 on a vendored shape once passed the + strict leg as "network". """ FETCH_SITES = ( "check_pins", "check_action_pins", - "check_userportal_profiles", "process_set", ) - def test_the_classifier_names_the_rate_limit_remedy(self): - body = _extract("classify_fetch_failure") - self.assertRegex(body, r"RATE-LIMITED[^\n]*GITHUB_TOKEN") - def test_every_fetch_site_uses_the_classifier_and_none_reimplements_it(self): from _helpers import strip_comments diff --git a/scripts/tests/test_vendored_shared_commit.py b/scripts/tests/test_vendored_shared_commit.py index 7f5823b..2de05b6 100755 --- a/scripts/tests/test_vendored_shared_commit.py +++ b/scripts/tests/test_vendored_shared_commit.py @@ -71,13 +71,6 @@ def test_a_missing_commit_line_fails_rather_than_matching_empty_to_empty(self): p = _run("", "") self.assertEqual(p.returncode, 1, p.stdout) - def test_the_shipped_script_calls_it_from_verify_and_check(self): - src = VENDORED.read_text(encoding="utf-8") - verify_body = src.split(" verify)", 1)[1].split(" drift)", 1)[0] - check_body = src.split(" check)", 1)[1].split(" verify)", 1)[0] - self.assertIn("check_shared_commit", verify_body) - self.assertIn("check_shared_commit", check_body) - if __name__ == "__main__": unittest.main() diff --git a/scripts/vendored.sh b/scripts/vendored.sh index 958edeb..dfa9bb8 100755 --- a/scripts/vendored.sh +++ b/scripts/vendored.sh @@ -53,22 +53,20 @@ # or SET matches nothing. News for Beacon and VRS, which the # node tracks by release; a finding for the gdi-metadata shapes, which is # why `ci-local.sh pins` runs `drift gdi-metadata`. Network. -# pins Assert the upstream content this repo depends on is still what it was -# pinned against: USERPORTAL_PROFILE_PINS (at the tags the userportal's -# ckan/Dockerfile names), EXTERNAL_PINS, and the workflows' action pins. -# `check` runs this too. +# pins Check that the upstream tokens this repo depends on are still what they +# were pinned against: EXTERNAL_PINS and the workflows' action pins. +# Exit 1 = drift, 2 = could not fetch (a network problem, not a pin +# problem). Not run by `check`. Network. # -# Scope: `check` and `fetch` verify the fidelity of the files this repo does vendor. They -# do not detect that upstream has added a file worth having, because each set vendors a -# curated subset (beacon omits `examples*/`, for one), so picking up new files stays a -# re-vendor decision rather than drift. `pins` covers the opposite case: upstream files -# this repo does not vendor but depends on, by hash where the content matters (the deployed -# profile code) and by substring where a token does (a release tag). `pins` makes one +# Scope: `check` and `fetch` verify the fidelity of the files this repo does vendor. They do +# not detect that upstream has added a file worth having, because each set vendors a curated +# subset (beacon omits `examples*/`, for one), so picking up new files stays a re-vendor +# decision rather than drift. `pins` covers the opposite case: upstream state this repo +# depends on but does not vendor, a release tag and the action tags. It makes one # `api.github.com` call for the beacon-v2 release watch and one per distinct Action pin, -# against an unauthenticated budget of 60/h per IP. Set GITHUB_TOKEN to raise it; -# the token is sent to api.github.com only (see curl_to). A rate-limited 403 is reported -# as RATE-LIMITED rather than a bare UNREACHABLE, so the remedy is named where the -# failure is. +# against an unauthenticated budget of 60/h per IP. Set GITHUB_TOKEN to raise it; the token +# is sent to api.github.com only (see curl_to). A rate-limited 403 is reported as +# RATE-LIMITED rather than a bare UNREACHABLE, so the remedy is named where the failure is. set -euo pipefail RAW="https://raw.githubusercontent.com" @@ -83,41 +81,17 @@ SETS=( "conformance/ga4gh-vrs-1.3" ) -# What the userportal's harvest reads from this node is decided by two files this repo does -# not vendor: the `fairdatapoint_dcat_ap` profile in the GDI fork of ckanext-fairdatapoint -# and its parent `euro_health_dcat_ap` in the GDI fork of ckanext-dcat, which -# `conformance/check_ckanext.py` hand-mirrors. The mirror holds while that code is what it -# was mirrored from; the conformance venv's `ckanext-dcat` pin holds while the dcat fork's -# upstream base is that version. Both are checked at whatever tag the userportal deploys, -# read from its own `ckan/Dockerfile`, so a userportal release that moves a tag without -# moving either passes. -USERPORTAL_DOCKERFILE="GenomicDataInfrastructure/gdi-userportal-ckan-docker/main|ckan/Dockerfile" -# Format: "|||