diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fbbb6be --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +--- +# PR gate: every pull request to main (and every push to a non-main branch) validates +# the committed sector data against the repo schemas before it can be merged. This is a +# data-only repo, so the check is a single self-contained validation script. +name: CI checks + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize] + push: + branches-ignore: [main] + workflow_dispatch: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install validation deps + run: python -m pip install --upgrade pip "jsonschema[format]" pyyaml pyarrow + + - name: Validate data against schemas + run: python scripts/validate.py diff --git a/README.md b/README.md index e4ba8d6..1bcf6cb 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ artifacts only, no fetch/parse/calculate code. ``` schema/ # YAML column definitions + JSON Schema for metadata data/ # raw parquet, one subfolder per sector / data type (ranked) +scripts/ # CI validation (scripts/validate.py — data vs schema contracts) ``` ## Data @@ -40,8 +41,10 @@ here schemas describe parquet columns directly. These files are the **contract**, not code. `sentier-importers` reads them to validate and cast incoming data before delivering parquet here. This repo ships -no Python package, loader, or tests — all ingestion logic lives in the importer, -the read side in `sentier-platform`. +no Python package or loader — all ingestion logic lives in the importer, the +read side in `sentier-platform`. CI enforces the contract on every PR: +`scripts/validate.py` checks each sector folder's parquet columns, enums, +row counts, `process_id` uniqueness/foreign keys, and `metadata.json`. ## License diff --git a/data/01-agriculture/exchanges.parquet b/data/01-agriculture/exchanges.parquet new file mode 100644 index 0000000..0d77d3d Binary files /dev/null and b/data/01-agriculture/exchanges.parquet differ diff --git a/data/01-agriculture/metadata.json b/data/01-agriculture/metadata.json index 07bb096..ae24205 100644 --- a/data/01-agriculture/metadata.json +++ b/data/01-agriculture/metadata.json @@ -1,8 +1,11 @@ { "sector": "agriculture", "title": "Agriculture", - "description": null, + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 1, "schema_version": "0.1.0", - "row_counts": {} + "row_counts": { + "processes.parquet": 210, + "exchanges.parquet": 7634 + } } diff --git a/data/01-agriculture/processes.parquet b/data/01-agriculture/processes.parquet new file mode 100644 index 0000000..6637559 Binary files /dev/null and b/data/01-agriculture/processes.parquet differ diff --git a/data/02-electricity/exchanges.parquet b/data/02-electricity/exchanges.parquet new file mode 100644 index 0000000..eaddd99 Binary files /dev/null and b/data/02-electricity/exchanges.parquet differ diff --git a/data/02-electricity/metadata.json b/data/02-electricity/metadata.json index caf23b3..21420ac 100644 --- a/data/02-electricity/metadata.json +++ b/data/02-electricity/metadata.json @@ -1,8 +1,11 @@ { "sector": "electricity", "title": "Electricity", - "description": null, + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 2, "schema_version": "0.1.0", - "row_counts": {} + "row_counts": { + "processes.parquet": 2033, + "exchanges.parquet": 39133 + } } diff --git a/data/02-electricity/processes.parquet b/data/02-electricity/processes.parquet new file mode 100644 index 0000000..95d41c5 Binary files /dev/null and b/data/02-electricity/processes.parquet differ diff --git a/data/03-chemicals/exchanges.parquet b/data/03-chemicals/exchanges.parquet new file mode 100644 index 0000000..f41d21b Binary files /dev/null and b/data/03-chemicals/exchanges.parquet differ diff --git a/data/03-chemicals/metadata.json b/data/03-chemicals/metadata.json index b0abdcb..c19214e 100644 --- a/data/03-chemicals/metadata.json +++ b/data/03-chemicals/metadata.json @@ -1,8 +1,11 @@ { "sector": "chemicals", - "title": "Chemicals & materials", - "description": null, + "title": "Chemicals", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 3, "schema_version": "0.1.0", - "row_counts": {} + "row_counts": { + "processes.parquet": 510, + "exchanges.parquet": 35566 + } } diff --git a/data/03-chemicals/processes.parquet b/data/03-chemicals/processes.parquet new file mode 100644 index 0000000..5eb3aae Binary files /dev/null and b/data/03-chemicals/processes.parquet differ diff --git a/data/04-transport/exchanges.parquet b/data/04-transport/exchanges.parquet new file mode 100644 index 0000000..79c2071 Binary files /dev/null and b/data/04-transport/exchanges.parquet differ diff --git a/data/04-transport/metadata.json b/data/04-transport/metadata.json new file mode 100644 index 0000000..120ad40 --- /dev/null +++ b/data/04-transport/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "transport", + "title": "Transport", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 4, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 2017, + "exchanges.parquet": 91780 + } +} diff --git a/data/04-transport/processes.parquet b/data/04-transport/processes.parquet new file mode 100644 index 0000000..d5e12e5 Binary files /dev/null and b/data/04-transport/processes.parquet differ diff --git a/data/05-energy/exchanges.parquet b/data/05-energy/exchanges.parquet new file mode 100644 index 0000000..7c77753 Binary files /dev/null and b/data/05-energy/exchanges.parquet differ diff --git a/data/05-energy/metadata.json b/data/05-energy/metadata.json new file mode 100644 index 0000000..2e1ed34 --- /dev/null +++ b/data/05-energy/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "energy", + "title": "Energy carriers", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 5, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 3188, + "exchanges.parquet": 46489 + } +} diff --git a/data/05-energy/processes.parquet b/data/05-energy/processes.parquet new file mode 100644 index 0000000..0922e7d Binary files /dev/null and b/data/05-energy/processes.parquet differ diff --git a/data/06-waste/exchanges.parquet b/data/06-waste/exchanges.parquet new file mode 100644 index 0000000..341ce1e Binary files /dev/null and b/data/06-waste/exchanges.parquet differ diff --git a/data/06-waste/metadata.json b/data/06-waste/metadata.json new file mode 100644 index 0000000..0e60c3b --- /dev/null +++ b/data/06-waste/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "waste", + "title": "Waste management", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 6, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 848, + "exchanges.parquet": 25777 + } +} diff --git a/data/06-waste/processes.parquet b/data/06-waste/processes.parquet new file mode 100644 index 0000000..b68c548 Binary files /dev/null and b/data/06-waste/processes.parquet differ diff --git a/data/07-construction/exchanges.parquet b/data/07-construction/exchanges.parquet new file mode 100644 index 0000000..d1acde9 Binary files /dev/null and b/data/07-construction/exchanges.parquet differ diff --git a/data/07-construction/metadata.json b/data/07-construction/metadata.json new file mode 100644 index 0000000..87ce934 --- /dev/null +++ b/data/07-construction/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "construction", + "title": "Construction", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 7, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 927, + "exchanges.parquet": 97366 + } +} diff --git a/data/07-construction/processes.parquet b/data/07-construction/processes.parquet new file mode 100644 index 0000000..c8762ed Binary files /dev/null and b/data/07-construction/processes.parquet differ diff --git a/data/08-materials/exchanges.parquet b/data/08-materials/exchanges.parquet new file mode 100644 index 0000000..2de7584 Binary files /dev/null and b/data/08-materials/exchanges.parquet differ diff --git a/data/08-materials/metadata.json b/data/08-materials/metadata.json new file mode 100644 index 0000000..2bdef7b --- /dev/null +++ b/data/08-materials/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "materials", + "title": "Materials", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 8, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 867, + "exchanges.parquet": 32386 + } +} diff --git a/data/08-materials/processes.parquet b/data/08-materials/processes.parquet new file mode 100644 index 0000000..1beba7a Binary files /dev/null and b/data/08-materials/processes.parquet differ diff --git a/data/09-electronics/exchanges.parquet b/data/09-electronics/exchanges.parquet new file mode 100644 index 0000000..306fe0d Binary files /dev/null and b/data/09-electronics/exchanges.parquet differ diff --git a/data/09-electronics/metadata.json b/data/09-electronics/metadata.json new file mode 100644 index 0000000..e673647 --- /dev/null +++ b/data/09-electronics/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "electronics", + "title": "Electronics", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 9, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 300, + "exchanges.parquet": 4553 + } +} diff --git a/data/09-electronics/processes.parquet b/data/09-electronics/processes.parquet new file mode 100644 index 0000000..8fda56a Binary files /dev/null and b/data/09-electronics/processes.parquet differ diff --git a/data/10-building-services/exchanges.parquet b/data/10-building-services/exchanges.parquet new file mode 100644 index 0000000..6e5bc70 Binary files /dev/null and b/data/10-building-services/exchanges.parquet differ diff --git a/data/10-building-services/metadata.json b/data/10-building-services/metadata.json new file mode 100644 index 0000000..898035b --- /dev/null +++ b/data/10-building-services/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "building-services", + "title": "Building services", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 10, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 268, + "exchanges.parquet": 11887 + } +} diff --git a/data/10-building-services/processes.parquet b/data/10-building-services/processes.parquet new file mode 100644 index 0000000..fc75788 Binary files /dev/null and b/data/10-building-services/processes.parquet differ diff --git a/data/99-obsolete/exchanges.parquet b/data/99-obsolete/exchanges.parquet new file mode 100644 index 0000000..e225cfb Binary files /dev/null and b/data/99-obsolete/exchanges.parquet differ diff --git a/data/99-obsolete/metadata.json b/data/99-obsolete/metadata.json new file mode 100644 index 0000000..9d8828f --- /dev/null +++ b/data/99-obsolete/metadata.json @@ -0,0 +1,11 @@ +{ + "sector": "obsolete", + "title": "Obsolete (BAFU legacy link targets)", + "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", + "rank": 99, + "schema_version": "0.1.0", + "row_counts": { + "processes.parquet": 779, + "exchanges.parquet": 27492 + } +} diff --git a/data/99-obsolete/processes.parquet b/data/99-obsolete/processes.parquet new file mode 100644 index 0000000..508af30 Binary files /dev/null and b/data/99-obsolete/processes.parquet differ diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..d84b504 --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Validate committed sector data against the repo schema contracts. + +Checks every ``data/-/`` folder: + +- ``metadata.json`` validates against ``schema/metadata.schema.json`` and its + ``row_counts`` match the actual parquet row counts; +- ``processes.parquet`` / ``exchanges.parquet`` conform to + ``schema/process.yaml`` / ``schema/exchange.yaml``: no unknown columns, + required columns present and non-null, column types match, enum columns only + hold allowed values; +- ``processes.process_id`` is unique per folder and every + ``exchanges.process_id`` resolves to a process in the same folder. + +Columns declared optional may be omitted entirely. A column whose values are +all null passes the type check regardless of its physical type (an all-null +column carries no typed values to disagree with the contract). + +Run from the repo root: ``python scripts/validate.py``. +Exits non-zero listing every violation found. +""" + +import json +import sys +from pathlib import Path + +import jsonschema +import pyarrow.compute as pc +import pyarrow.parquet as pq +import yaml + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_DIR = ROOT / "schema" +DATA_DIR = ROOT / "data" + +# schema type -> acceptable pyarrow type names +TYPE_MAP = { + "string": {"string", "large_string"}, + "double": {"double", "float"}, + "int": {"int64", "int32", "int16", "int8"}, + "date": {"date32[day]", "date64[ms]", "timestamp[ms]", "timestamp[us]", "timestamp[ns]"}, +} + + +def load_table_schema(name): + spec = yaml.safe_load((SCHEMA_DIR / name).read_text()) + return {col["name"]: col for col in spec["columns"]} + + +def validate_table(folder, table_name, columns, enums, errors): + path = folder / f"{table_name}.parquet" + label = f"{folder.name}/{path.name}" + if not path.exists(): + errors.append(f"{label}: file missing") + return None + table = pq.read_table(path) + + unknown = set(table.column_names) - set(columns) + if unknown: + errors.append(f"{label}: unknown columns {sorted(unknown)}") + + for name, col in columns.items(): + if name not in table.column_names: + if col.get("required"): + errors.append(f"{label}: required column '{name}' missing") + continue + data = table[name] + nulls = data.null_count + if col.get("required") and nulls: + errors.append(f"{label}: required column '{name}' has {nulls} nulls") + if nulls == len(data): + continue # all-null: no typed values to check + expected = TYPE_MAP.get(col["type"]) if col["type"] != "enum" else TYPE_MAP["string"] + if expected and str(data.type) not in expected: + errors.append(f"{label}: column '{name}' is {data.type}, expected {col['type']}") + if col["type"] == "enum": + allowed = set(enums[col["enum"]]) + seen = set(pc.drop_null(data.combine_chunks()).unique().to_pylist()) + bad = seen - allowed + if bad: + errors.append(f"{label}: column '{name}' has values outside enum {col['enum']}: {sorted(bad)}") + return table + + +def validate_folder(folder, process_cols, exchange_cols, enums, metadata_schema, errors): + meta_path = folder / "metadata.json" + if not meta_path.exists(): + errors.append(f"{folder.name}: metadata.json missing") + meta = {} + else: + meta = json.loads(meta_path.read_text()) + try: + jsonschema.validate(meta, metadata_schema) + except jsonschema.ValidationError as exc: + errors.append(f"{folder.name}/metadata.json: {exc.message}") + + processes = validate_table(folder, "processes", process_cols, enums, errors) + exchanges = validate_table(folder, "exchanges", exchange_cols, enums, errors) + + for name, table in (("processes", processes), ("exchanges", exchanges)): + declared = meta.get("row_counts", {}).get(f"{name}.parquet") + if table is not None and declared is not None and declared != len(table): + errors.append( + f"{folder.name}/metadata.json: row_counts says {name}.parquet has " + f"{declared} rows, actual {len(table)}" + ) + + if processes is not None and "process_id" in processes.column_names: + ids = processes["process_id"] + n_unique = len(pc.drop_null(ids.combine_chunks()).unique()) + if n_unique != len(ids): + errors.append(f"{folder.name}/processes.parquet: process_id not unique") + if exchanges is not None and "process_id" in exchanges.column_names: + known = set(ids.to_pylist()) + referenced = set(pc.drop_null(exchanges["process_id"].combine_chunks()).unique().to_pylist()) + orphans = referenced - known + if orphans: + errors.append( + f"{folder.name}/exchanges.parquet: {len(orphans)} process_id values " + f"with no process row (e.g. {sorted(orphans)[:3]})" + ) + + +def main(): + process_cols = load_table_schema("process.yaml") + exchange_cols = load_table_schema("exchange.yaml") + enums = yaml.safe_load((SCHEMA_DIR / "common.yaml").read_text())["enums"] + metadata_schema = json.loads((SCHEMA_DIR / "metadata.schema.json").read_text()) + + folders = sorted(p for p in DATA_DIR.iterdir() if p.is_dir()) + if not folders: + print("No sector folders under data/ — nothing to validate.") + return 0 + + errors = [] + for folder in folders: + validate_folder(folder, process_cols, exchange_cols, enums, metadata_schema, errors) + + if errors: + print(f"FAIL — {len(errors)} violation(s):") + for err in errors: + print(f" - {err}") + return 1 + print(f"OK — {len(folders)} sector folder(s) validate against the schema contracts.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())