diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbbb6be..d00e19b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ --- # PR gate: every pull request to main (and every push to a non-main branch) validates -# the committed sector data against the repo schemas before it can be merged. This is a +# the committed source data against the repo schemas before it can be merged. This is a # data-only repo, so the check is a single self-contained validation script. name: CI checks diff --git a/README.md b/README.md index c46be0b..e0dba25 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Standardized life cycle inventory (LCI) data for the Sentier platform. Processes and their exchanges, stored as parquet. A Data Layer repo: artifacts only. - Fed by `sentier-importers`, which opens PRs with validated parquet. -- Read by `sentier-brightway`, which loads every sector folder. +- Read by `sentier-brightway`, which loads one source folder into one Brightway database. - Cross-source flow ids resolve via `sentier-mappings`. - No Python package, no loader. The only code is the CI validator. @@ -16,7 +16,7 @@ Nothing to install. Clone the repo and read the parquet files directly. ## Use -Validate all sector data against the schema contracts. This is what CI runs on every PR. +Validate all source folders against the schema contracts. This is what CI runs on every PR. ```bash uv run --with "jsonschema[format],pyyaml,pyarrow" python scripts/validate.py @@ -25,33 +25,32 @@ uv run --with "jsonschema[format],pyyaml,pyarrow" python scripts/validate.py Inspect a delivered parquet payload. ```bash -uv run --with pandas,pyarrow python -c "import pandas as pd; print(pd.read_parquet('data/01-agriculture/processes.parquet'))" +uv run --with pandas,pyarrow python -c "import pandas as pd; print(pd.read_parquet('data/bafu-2026-v1/processes.parquet'))" ``` ## Layout ``` schema/ # the data contract, read by sentier-importers -data/-/ # one folder per sector, ranked by NN +data/-/ # one folder per published release of one datasource scripts/validate.py # CI validator, self-contained .github/workflows/ci.yml # runs scripts/validate.py on every PR and branch push ``` ## Data -Inventory is organized by sector, not by source. Each process row carries a `source` tag (`bafu-2026`) and each folder's `metadata.json` lists the `sources` it holds, so consumers select by source without a source-named folder. See [data/README.md](data/README.md) for the id convention. +Inventory is organized by source. One folder per published release of one datasource, named `-` (`bafu-2026-v1`). A delivery replaces its source folder whole; sources never share a file. Every process row carries `source`, `source_version` and `sector`, so a consumer picks a folder by its `metadata.json` and filters rows by sector. See [data/README.md](data/README.md) for the id convention. -Each `data/-/` folder holds: +Each `data/-/` folder holds: | file | rows | |---|---| | `processes.parquet` | one per process | | `exchanges.parquet` | one per exchange (technosphere and biosphere edges) | -| `metadata.json` | sector, title, rank, schema_version, sources, row_counts | +| `metadata.json` | source, source_version, title, publisher, license, citation, schema_version, sectors, row_counts | -- The `NN` prefix orders sectors. Lower wins when records overlap. -- Sector ids are lower-kebab: `agriculture`, `electricity`, `chemicals`. -- Background link targets such as ecoinvent are not sectors. They resolve through `sentier-mappings`. +- Sector ids are lower-kebab enum values in `schema/common.yaml`: `agriculture`, `electricity`, `chemicals`, ... +- Background link targets such as ecoinvent are not sources here. They resolve through `sentier-mappings`. - Parquet is committed directly. No git-LFS, no release artifacts. ## Schema @@ -60,12 +59,12 @@ Plain YAML. Each file lists the columns of one parquet table. LinkML is used onl | file | describes | |---|---| -| `schema/common.yaml` | shared enums: `flow_direction`, `flow_type`, `process_type` | +| `schema/common.yaml` | shared enums: `flow_direction`, `flow_type`, `process_type`, `sector` | | `schema/process.yaml` | `processes.parquet`, primary key `process_id` | | `schema/exchange.yaml` | `exchanges.parquet`, joins on `process_id` | -| `schema/metadata.schema.json` | JSON Schema for each sector's `metadata.json` | +| `schema/metadata.schema.json` | JSON Schema for each source folder's `metadata.json` | -CI checks columns, types, enums, `row_counts`, `process_id` uniqueness, and the exchange to process foreign key per sector. +CI checks columns, types, enums, `row_counts`, `process_id` uniqueness, the exchange to process foreign key, one `source`/`source_version` per folder matching its name, and `sectors` against the column. ## Contributing diff --git a/data/01-agriculture/.gitkeep b/data/01-agriculture/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/01-agriculture/exchanges.parquet b/data/01-agriculture/exchanges.parquet deleted file mode 100644 index d61667d..0000000 Binary files a/data/01-agriculture/exchanges.parquet and /dev/null differ diff --git a/data/01-agriculture/metadata.json b/data/01-agriculture/metadata.json deleted file mode 100644 index b73e4b8..0000000 --- a/data/01-agriculture/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "agriculture", - "title": "Agriculture", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 1, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 210, - "exchanges.parquet": 7634 - } -} diff --git a/data/01-agriculture/processes.parquet b/data/01-agriculture/processes.parquet deleted file mode 100644 index 3a7bce4..0000000 Binary files a/data/01-agriculture/processes.parquet and /dev/null differ diff --git a/data/02-electricity/.gitkeep b/data/02-electricity/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/02-electricity/exchanges.parquet b/data/02-electricity/exchanges.parquet deleted file mode 100644 index cc0ea07..0000000 Binary files a/data/02-electricity/exchanges.parquet and /dev/null differ diff --git a/data/02-electricity/metadata.json b/data/02-electricity/metadata.json deleted file mode 100644 index ceb9eaf..0000000 --- a/data/02-electricity/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "electricity", - "title": "Electricity", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 2, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 2033, - "exchanges.parquet": 39133 - } -} diff --git a/data/02-electricity/processes.parquet b/data/02-electricity/processes.parquet deleted file mode 100644 index 27bf57c..0000000 Binary files a/data/02-electricity/processes.parquet and /dev/null differ diff --git a/data/03-chemicals/.gitkeep b/data/03-chemicals/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/03-chemicals/exchanges.parquet b/data/03-chemicals/exchanges.parquet deleted file mode 100644 index ff1860d..0000000 Binary files a/data/03-chemicals/exchanges.parquet and /dev/null differ diff --git a/data/03-chemicals/metadata.json b/data/03-chemicals/metadata.json deleted file mode 100644 index 4bfbb30..0000000 --- a/data/03-chemicals/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "chemicals", - "title": "Chemicals", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 3, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 510, - "exchanges.parquet": 35566 - } -} diff --git a/data/03-chemicals/processes.parquet b/data/03-chemicals/processes.parquet deleted file mode 100644 index 9c1b2bc..0000000 Binary files a/data/03-chemicals/processes.parquet and /dev/null differ diff --git a/data/04-transport/exchanges.parquet b/data/04-transport/exchanges.parquet deleted file mode 100644 index a08f256..0000000 Binary files a/data/04-transport/exchanges.parquet and /dev/null differ diff --git a/data/04-transport/metadata.json b/data/04-transport/metadata.json deleted file mode 100644 index 84035eb..0000000 --- a/data/04-transport/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "transport", - "title": "Transport", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 4, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 2017, - "exchanges.parquet": 91780 - } -} diff --git a/data/04-transport/processes.parquet b/data/04-transport/processes.parquet deleted file mode 100644 index 6ca7c84..0000000 Binary files a/data/04-transport/processes.parquet and /dev/null differ diff --git a/data/05-energy/exchanges.parquet b/data/05-energy/exchanges.parquet deleted file mode 100644 index e33a6b2..0000000 Binary files a/data/05-energy/exchanges.parquet and /dev/null differ diff --git a/data/05-energy/metadata.json b/data/05-energy/metadata.json deleted file mode 100644 index 137c445..0000000 --- a/data/05-energy/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "energy", - "title": "Energy carriers", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 5, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 3188, - "exchanges.parquet": 46489 - } -} diff --git a/data/05-energy/processes.parquet b/data/05-energy/processes.parquet deleted file mode 100644 index 0725a8b..0000000 Binary files a/data/05-energy/processes.parquet and /dev/null differ diff --git a/data/06-waste/exchanges.parquet b/data/06-waste/exchanges.parquet deleted file mode 100644 index 11f1a6d..0000000 Binary files a/data/06-waste/exchanges.parquet and /dev/null differ diff --git a/data/06-waste/metadata.json b/data/06-waste/metadata.json deleted file mode 100644 index 598b215..0000000 --- a/data/06-waste/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "waste", - "title": "Waste management", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 6, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 848, - "exchanges.parquet": 25777 - } -} diff --git a/data/06-waste/processes.parquet b/data/06-waste/processes.parquet deleted file mode 100644 index 5bcbb79..0000000 Binary files a/data/06-waste/processes.parquet and /dev/null differ diff --git a/data/07-construction/exchanges.parquet b/data/07-construction/exchanges.parquet deleted file mode 100644 index 2ea6e80..0000000 Binary files a/data/07-construction/exchanges.parquet and /dev/null differ diff --git a/data/07-construction/metadata.json b/data/07-construction/metadata.json deleted file mode 100644 index 3f045b2..0000000 --- a/data/07-construction/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "construction", - "title": "Construction", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 7, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 927, - "exchanges.parquet": 97366 - } -} diff --git a/data/07-construction/processes.parquet b/data/07-construction/processes.parquet deleted file mode 100644 index 8e5b009..0000000 Binary files a/data/07-construction/processes.parquet and /dev/null differ diff --git a/data/08-materials/exchanges.parquet b/data/08-materials/exchanges.parquet deleted file mode 100644 index f25908f..0000000 Binary files a/data/08-materials/exchanges.parquet and /dev/null differ diff --git a/data/08-materials/metadata.json b/data/08-materials/metadata.json deleted file mode 100644 index 11cb64a..0000000 --- a/data/08-materials/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "materials", - "title": "Materials", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 8, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 867, - "exchanges.parquet": 32386 - } -} diff --git a/data/08-materials/processes.parquet b/data/08-materials/processes.parquet deleted file mode 100644 index 250a2e9..0000000 Binary files a/data/08-materials/processes.parquet and /dev/null differ diff --git a/data/09-electronics/exchanges.parquet b/data/09-electronics/exchanges.parquet deleted file mode 100644 index bbebb77..0000000 Binary files a/data/09-electronics/exchanges.parquet and /dev/null differ diff --git a/data/09-electronics/metadata.json b/data/09-electronics/metadata.json deleted file mode 100644 index 9b2a1c4..0000000 --- a/data/09-electronics/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "electronics", - "title": "Electronics", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 9, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 300, - "exchanges.parquet": 4553 - } -} diff --git a/data/09-electronics/processes.parquet b/data/09-electronics/processes.parquet deleted file mode 100644 index cd86dd1..0000000 Binary files a/data/09-electronics/processes.parquet and /dev/null differ diff --git a/data/10-building-services/exchanges.parquet b/data/10-building-services/exchanges.parquet deleted file mode 100644 index a5eea4c..0000000 Binary files a/data/10-building-services/exchanges.parquet and /dev/null differ diff --git a/data/10-building-services/metadata.json b/data/10-building-services/metadata.json deleted file mode 100644 index e144636..0000000 --- a/data/10-building-services/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "building-services", - "title": "Building services", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 10, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 268, - "exchanges.parquet": 11887 - } -} diff --git a/data/10-building-services/processes.parquet b/data/10-building-services/processes.parquet deleted file mode 100644 index ead548e..0000000 Binary files a/data/10-building-services/processes.parquet and /dev/null differ diff --git a/data/99-obsolete/exchanges.parquet b/data/99-obsolete/exchanges.parquet deleted file mode 100644 index a24d6e3..0000000 Binary files a/data/99-obsolete/exchanges.parquet and /dev/null differ diff --git a/data/99-obsolete/metadata.json b/data/99-obsolete/metadata.json deleted file mode 100644 index 1aa82b3..0000000 --- a/data/99-obsolete/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sector": "obsolete", - "title": "Obsolete (BAFU legacy link targets)", - "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", - "rank": 99, - "schema_version": "0.2.0", - "sources": [ - "bafu-2026" - ], - "row_counts": { - "processes.parquet": 779, - "exchanges.parquet": 27492 - } -} diff --git a/data/99-obsolete/processes.parquet b/data/99-obsolete/processes.parquet deleted file mode 100644 index ce328cb..0000000 Binary files a/data/99-obsolete/processes.parquet and /dev/null differ diff --git a/data/README.md b/data/README.md index 3dbc357..b4752ab 100644 --- a/data/README.md +++ b/data/README.md @@ -1,26 +1,28 @@ # data/ -Raw **parquet** inventory artifacts, **one subfolder per sector / data type**, -ranked by a numeric prefix: +Raw **parquet** inventory artifacts, **one subfolder per published release of one +datasource**: ``` data/ - 01-agriculture/ # rank 1 — highest resolution precedence - 02-electricity/ # Brightcon deliverable - 03-chemicals/ # chemicals & materials + bafu-2026-v1/ # BAFU:2026 v1 LCI database ``` -Convention: `data/-/`, where `` is a lower-kebab sector / -data-type id (`agriculture`, `electricity`, `chemicals`) and `` sets folder -ordering / resolution precedence (lower wins when records overlap). - -Inventory is organized by **sector, not by datasource**. Upstream sources -(BAFU, Agribalyse, …) each span many sectors, so a folder may hold rows from -several sources. Every process row is tagged with its `source` (and optional -`source_version`), and each folder's `metadata.json` lists the `sources` present, -so a consumer can pick folders by metadata and rows by column. Precedence between -folders is by rank; within a folder the consumer selects by `source` (there is no -intra-folder precedence rule). The import log of which upstream file produced +Convention: `data/-/`, where `` is the lower-kebab +datasource id `-` and `` is the publisher's release. +A folder holds exactly one source and one version. A delivery from `sentier-importers` +replaces the folder whole; sources never share a file, so nothing is ever merged. A +re-release is a new folder; the PR that adds it removes the old one unless both are meant +to be served. + +Inventory is organized by **source**, with the **sector as a column**. Every process +row carries `source`, `source_version` and `sector` (enum in `../schema/common.yaml`: +`agriculture`, `electricity`, `chemicals`, `transport`, `energy`, `waste`, +`construction`, `materials`, `electronics`, `building-services`, `obsolete`). Each +folder's `metadata.json` carries the publisher, licence, required citation, the +`sectors` present and the row counts, so a consumer can pick a folder from metadata +alone and filter rows by sector. Within one source there is no precedence rule; +between sources the consumer chooses. The import log of which upstream file produced which rows stays in `sentier-importers`. ### Source ids @@ -31,15 +33,12 @@ sentier-mappings pair-folder prefix. The publisher's release goes in `source_version`, never in the id, so a re-release does not rename the database a consumer's project depends on. -| datasource | `source` | `source_version` | -|---|---|---| -| BAFU:2026 v1 LCI database | `bafu-2026` | `v1` | +| datasource | `source` | `source_version` | folder | +|---|---|---|---| +| BAFU:2026 v1 LCI database | `bafu-2026` | `v1` | `bafu-2026-v1/` | > Background datasets that exist only as link targets (e.g. ecoinvent) are not a -> sector; they are resolved through `sentier-mappings`, not stored here as their -> own folder. +> source here; they are resolved through `sentier-mappings`, not stored as a folder. Each folder holds `processes.parquet`, `exchanges.parquet`, and `metadata.json`. -See [../schema/](../schema/) for column definitions. Parquet payloads are pushed -by `sentier-importers` / `sentier-models`; the skeleton ships `.gitkeep` + -`metadata.json` stubs only. +See [../schema/](../schema/) for column definitions. diff --git a/data/bafu-2026-v1/exchanges.parquet b/data/bafu-2026-v1/exchanges.parquet new file mode 100644 index 0000000..df71922 Binary files /dev/null and b/data/bafu-2026-v1/exchanges.parquet differ diff --git a/data/bafu-2026-v1/metadata.json b/data/bafu-2026-v1/metadata.json new file mode 100644 index 0000000..508580c --- /dev/null +++ b/data/bafu-2026-v1/metadata.json @@ -0,0 +1,29 @@ +{ + "source": "bafu-2026", + "source_version": "v1", + "title": "Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026 (v1)", + "publisher": "Swiss Federal Office for the Environment (BAFU/FOEN)", + "license": "BAFU Terms of Use: free use and processing, no redistribution as or inside another database without agreement (§2.1/§2.2), share-alike and change documentation for modified data (§3), citation required (§2.3)", + "citation": "Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026. Version 1, EcoSpold v1 export (11,947 datasets).", + "source_url": null, + "provenance": "https://vocab.sentier.dev/sources/bafu-2026", + "retrieved": null, + "schema_version": "0.3.0", + "sectors": [ + "agriculture", + "building-services", + "chemicals", + "construction", + "electricity", + "electronics", + "energy", + "materials", + "obsolete", + "transport", + "waste" + ], + "row_counts": { + "processes.parquet": 11947, + "exchanges.parquet": 420063 + } +} diff --git a/data/bafu-2026-v1/processes.parquet b/data/bafu-2026-v1/processes.parquet new file mode 100644 index 0000000..5fc9b3a Binary files /dev/null and b/data/bafu-2026-v1/processes.parquet differ diff --git a/schema/README.md b/schema/README.md index d927ddb..5ee55e5 100644 --- a/schema/README.md +++ b/schema/README.md @@ -7,10 +7,10 @@ parquet repos describe columns directly. | file | describes | |---|---| -| `common.yaml` | shared enums (`flow_direction`, `flow_type`, `process_type`) | +| `common.yaml` | shared enums (`flow_direction`, `flow_type`, `process_type`, `sector`) | | `process.yaml` | `processes.parquet` | | `exchange.yaml` | `exchanges.parquet` | -| `metadata.schema.json` | JSON Schema for each sector's `metadata.json` | +| `metadata.schema.json` | JSON Schema for each source folder's `metadata.json` | Column types: `string` · `double` · `int` · `date` · `enum` (enum values live in `common.yaml`). `processes.parquet` is the parent; `exchanges.parquet` joins on diff --git a/schema/common.yaml b/schema/common.yaml index 0cd82e6..f8a79f1 100644 --- a/schema/common.yaml +++ b/schema/common.yaml @@ -16,4 +16,17 @@ enums: - system # a system process aggregating multiple unit processes - lci_result # a cradle-to-gate LCI result + sector: # sector / data-type a process belongs to (was the folder level < 0.3.0) + - agriculture + - electricity + - chemicals + - transport + - energy + - waste + - construction + - materials + - electronics + - building-services + - obsolete # legacy link targets kept so older references resolve + # Column types used across schemas: string | double | int | date | enum diff --git a/schema/metadata.schema.json b/schema/metadata.schema.json index f76779a..a155e1c 100644 --- a/schema/metadata.schema.json +++ b/schema/metadata.schema.json @@ -1,45 +1,78 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://data.sentier.dev/inventory/schema/metadata", - "title": "Sentier inventory sector metadata", - "description": "Provenance file (metadata.json) carried by each data/-/ folder. Inventory is organised by sector; \"sources\" lists every datasource id present in this folder's processes.parquet \"source\" column. The source-to-row import log stays in sentier-importers.", + "title": "Sentier inventory source metadata", + "description": "Provenance file (metadata.json) carried by each data/-/ folder. One folder holds exactly one published release of one datasource; sector is a column on processes.parquet. The source-to-row import log stays in sentier-importers.", "type": "object", "required": [ - "sector", + "source", + "source_version", "title", - "rank", + "publisher", + "license", + "citation", "schema_version", - "sources" + "sectors", + "row_counts" ], "additionalProperties": true, "properties": { - "sector": { + "source": { "type": "string", - "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals." + "pattern": "^[a-z0-9]+(-[a-z0-9.]+)*$", + "description": "Datasource id, lower-kebab -, e.g. bafu-2026. Equals the single distinct processes.source value (CI-checked)." + }, + "source_version": { + "type": "string", + "pattern": "^[a-z0-9]+([.-][a-z0-9]+)*$", + "description": "Publisher's release of that dataset, e.g. v1. Equals the single distinct processes.source_version value; the folder is named - (CI-checked)." }, "title": { "type": "string" }, - "description": { + "publisher": { "type": "string" }, - "rank": { - "type": "integer", - "minimum": 1, - "description": "Folder ordering / resolution precedence; lower wins." + "license": { + "type": "string", + "description": "Licence name, URL or terms-of-use summary as the publisher states it." + }, + "citation": { + "type": "string", + "description": "The citation sentence the publisher requires." + }, + "source_url": { + "type": [ + "string", + "null" + ], + "description": "Where the upstream was obtained." + }, + "provenance": { + "type": [ + "string", + "null" + ], + "description": "sentier-vocab Source IRI of this datasource." + }, + "retrieved": { + "type": [ + "string", + "null" + ], + "format": "date" }, "schema_version": { "type": "string" }, - "sources": { + "sectors": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { - "type": "string", - "pattern": "^[a-z0-9]+(-[a-z0-9.]+)*$" + "type": "string" }, - "description": "Datasource ids present in this folder's processes.parquet source column, e.g. [\"bafu-2026\"]. Must equal the column's distinct values (CI-checked)." + "description": "Sorted distinct processes.sector values present in this folder (CI-checked)." }, "row_counts": { "type": "object", diff --git a/schema/process.yaml b/schema/process.yaml index eb966c4..42d47f7 100644 --- a/schema/process.yaml +++ b/schema/process.yaml @@ -38,19 +38,27 @@ columns: type: enum enum: process_type # see common.yaml required: true + - name: sector + type: enum + enum: sector # see common.yaml + required: true + description: > + Sector / data-type id the process belongs to. Replaces the former folder level; + the publisher-category to sector map lives in sentier-importers. - name: source type: string required: true description: > Datasource id of the upstream dataset this row was imported from, lower-kebab - -, e.g. bafu-2026. Must appear in the sector's - metadata.json "sources" list. Consumers select rows by this column. + -, e.g. bafu-2026. One value per folder, equal to the + folder's metadata.json "source". Consumers select a source by this id. - name: source_version type: string - required: false + required: true description: > - Publisher's release/version of that dataset, e.g. v1. Free-form; never part of - the source id. + Publisher's release/version of that dataset, e.g. v1. Never part of the source + id. One value per folder, equal to metadata.json "source_version"; the folder is + named -. - name: technology type: string required: false diff --git a/scripts/validate.py b/scripts/validate.py index cae86dd..ce6f6a2 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -"""Validate committed sector data against the repo schema contracts. +"""Validate committed source data against the repo schema contracts. -Checks every ``data/-/`` folder: +Checks every ``data/-/`` folder: - ``metadata.json`` validates against ``schema/metadata.schema.json`` and its ``row_counts`` match the actual parquet row counts; @@ -11,7 +11,10 @@ hold allowed values; - ``processes.process_id`` is unique per folder and every ``exchanges.process_id`` resolves to a process in the same folder; -- the distinct ``processes.source`` values equal ``metadata.json`` ``sources`` (both +- ``processes.source`` and ``processes.source_version`` each hold exactly one value, + equal to ``metadata.json`` ``source`` / ``source_version``, and the folder is named + ``-``; +- the distinct ``processes.sector`` values equal ``metadata.json`` ``sectors`` (both directions), so consumers can select folders by metadata without opening parquet. Columns declared optional may be omitted entirely. A column whose values are @@ -122,24 +125,56 @@ def validate_folder(folder, process_cols, exchange_cols, enums, metadata_schema, f"with no process row (e.g. {sorted(orphans)[:3]})" ) - if processes is not None and "source" in processes.column_names and "sources" in meta: - validate_sources(folder, processes["source"], meta["sources"], errors) + if processes is not None: + validate_source_identity(folder, processes, meta, errors) + if "sector" in processes.column_names and "sectors" in meta: + validate_sectors(folder, processes["sector"], meta["sectors"], errors) -def validate_sources(folder, source_column, declared, errors): - """``metadata.json`` ``sources`` must equal the distinct ``processes.source`` values.""" - seen = set(pc.drop_null(source_column.combine_chunks()).unique().to_pylist()) +def _distinct(column): + return set(pc.drop_null(column.combine_chunks()).unique().to_pylist()) + + +def validate_source_identity(folder, processes, meta, errors): + """One source and one version per folder, agreeing with metadata and the folder name.""" + for key in ("source", "source_version"): + if key not in processes.column_names: + continue + seen = _distinct(processes[key]) + if len(seen) != 1: + errors.append( + f"{folder.name}/processes.parquet: expected exactly one {key} value, " + f"found {sorted(seen)}" + ) + declared = meta.get(key) + if declared is not None and seen != {declared}: + errors.append( + f"{folder.name}/processes.parquet: {key} values {sorted(seen)} differ from " + f"metadata.json {key} {declared!r}" + ) + if "source" in meta and "source_version" in meta: + expected = f"{meta['source']}-{meta['source_version']}" + if folder.name != expected: + errors.append( + f"{folder.name}/metadata.json: folder should be named {expected!r} " + f"(-)" + ) + + +def validate_sectors(folder, sector_column, declared, errors): + """``metadata.json`` ``sectors`` must equal the distinct ``processes.sector`` values.""" + seen = _distinct(sector_column) declared = set(declared) undeclared = seen - declared if undeclared: errors.append( - f"{folder.name}/processes.parquet: source values not declared in " - f"metadata.json sources: {sorted(undeclared)}" + f"{folder.name}/processes.parquet: sector values not declared in " + f"metadata.json sectors: {sorted(undeclared)}" ) absent = declared - seen if absent: errors.append( - f"{folder.name}/metadata.json: sources declared but absent from " + f"{folder.name}/metadata.json: sectors declared but absent from " f"processes.parquet: {sorted(absent)}" ) @@ -152,7 +187,7 @@ def main(): folders = sorted(p for p in DATA_DIR.iterdir() if p.is_dir()) if not folders: - print("No sector folders under data/ — nothing to validate.") + print("No source folders under data/ — nothing to validate.") return 0 errors = [] @@ -164,7 +199,7 @@ def main(): for err in errors: print(f" - {err}") return 1 - print(f"OK — {len(folders)} sector folder(s) validate against the schema contracts.") + print(f"OK — {len(folders)} source folder(s) validate against the schema contracts.") return 0