diff --git a/README.md b/README.md index 5d05044..c46be0b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ scripts/validate.py # CI validator, self-contained ## Data -Inventory is organized by sector, not by source. Which source supplied which rows is logged in `sentier-importers`. +Inventory is organized by sector, not by source. Each process row carries a `source` tag (`bafu-2026`) and each folder's `metadata.json` lists the `sources` it holds, so consumers select by source without a source-named folder. See [data/README.md](data/README.md) for the id convention. Each `data/-/` folder holds: @@ -47,7 +47,7 @@ Each `data/-/` folder holds: |---|---| | `processes.parquet` | one per process | | `exchanges.parquet` | one per exchange (technosphere and biosphere edges) | -| `metadata.json` | sector, title, rank, schema_version, row_counts | +| `metadata.json` | sector, title, rank, schema_version, sources, row_counts | - The `NN` prefix orders sectors. Lower wins when records overlap. - Sector ids are lower-kebab: `agriculture`, `electricity`, `chemicals`. diff --git a/data/01-agriculture/metadata.json b/data/01-agriculture/metadata.json index ae24205..b73e4b8 100644 --- a/data/01-agriculture/metadata.json +++ b/data/01-agriculture/metadata.json @@ -3,7 +3,10 @@ "title": "Agriculture", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 1, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 210, "exchanges.parquet": 7634 diff --git a/data/01-agriculture/processes.parquet b/data/01-agriculture/processes.parquet index 6637559..3a7bce4 100644 Binary files a/data/01-agriculture/processes.parquet and b/data/01-agriculture/processes.parquet differ diff --git a/data/02-electricity/metadata.json b/data/02-electricity/metadata.json index 21420ac..ceb9eaf 100644 --- a/data/02-electricity/metadata.json +++ b/data/02-electricity/metadata.json @@ -3,7 +3,10 @@ "title": "Electricity", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 2, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 2033, "exchanges.parquet": 39133 diff --git a/data/02-electricity/processes.parquet b/data/02-electricity/processes.parquet index 95d41c5..27bf57c 100644 Binary files a/data/02-electricity/processes.parquet and b/data/02-electricity/processes.parquet differ diff --git a/data/03-chemicals/metadata.json b/data/03-chemicals/metadata.json index c19214e..4bfbb30 100644 --- a/data/03-chemicals/metadata.json +++ b/data/03-chemicals/metadata.json @@ -3,7 +3,10 @@ "title": "Chemicals", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 3, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 510, "exchanges.parquet": 35566 diff --git a/data/03-chemicals/processes.parquet b/data/03-chemicals/processes.parquet index 5eb3aae..9c1b2bc 100644 Binary files a/data/03-chemicals/processes.parquet and b/data/03-chemicals/processes.parquet differ diff --git a/data/04-transport/metadata.json b/data/04-transport/metadata.json index 120ad40..84035eb 100644 --- a/data/04-transport/metadata.json +++ b/data/04-transport/metadata.json @@ -3,7 +3,10 @@ "title": "Transport", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 4, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 2017, "exchanges.parquet": 91780 diff --git a/data/04-transport/processes.parquet b/data/04-transport/processes.parquet index d5e12e5..6ca7c84 100644 Binary files a/data/04-transport/processes.parquet and b/data/04-transport/processes.parquet differ diff --git a/data/05-energy/metadata.json b/data/05-energy/metadata.json index 2e1ed34..137c445 100644 --- a/data/05-energy/metadata.json +++ b/data/05-energy/metadata.json @@ -3,7 +3,10 @@ "title": "Energy carriers", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 5, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 3188, "exchanges.parquet": 46489 diff --git a/data/05-energy/processes.parquet b/data/05-energy/processes.parquet index 0922e7d..0725a8b 100644 Binary files a/data/05-energy/processes.parquet and b/data/05-energy/processes.parquet differ diff --git a/data/06-waste/metadata.json b/data/06-waste/metadata.json index 0e60c3b..598b215 100644 --- a/data/06-waste/metadata.json +++ b/data/06-waste/metadata.json @@ -3,7 +3,10 @@ "title": "Waste management", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 6, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 848, "exchanges.parquet": 25777 diff --git a/data/06-waste/processes.parquet b/data/06-waste/processes.parquet index b68c548..5bcbb79 100644 Binary files a/data/06-waste/processes.parquet and b/data/06-waste/processes.parquet differ diff --git a/data/07-construction/metadata.json b/data/07-construction/metadata.json index 87ce934..3f045b2 100644 --- a/data/07-construction/metadata.json +++ b/data/07-construction/metadata.json @@ -3,7 +3,10 @@ "title": "Construction", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 7, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 927, "exchanges.parquet": 97366 diff --git a/data/07-construction/processes.parquet b/data/07-construction/processes.parquet index c8762ed..8e5b009 100644 Binary files a/data/07-construction/processes.parquet and b/data/07-construction/processes.parquet differ diff --git a/data/08-materials/metadata.json b/data/08-materials/metadata.json index 2bdef7b..11cb64a 100644 --- a/data/08-materials/metadata.json +++ b/data/08-materials/metadata.json @@ -3,7 +3,10 @@ "title": "Materials", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 8, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 867, "exchanges.parquet": 32386 diff --git a/data/08-materials/processes.parquet b/data/08-materials/processes.parquet index 1beba7a..250a2e9 100644 Binary files a/data/08-materials/processes.parquet and b/data/08-materials/processes.parquet differ diff --git a/data/09-electronics/metadata.json b/data/09-electronics/metadata.json index e673647..9b2a1c4 100644 --- a/data/09-electronics/metadata.json +++ b/data/09-electronics/metadata.json @@ -3,7 +3,10 @@ "title": "Electronics", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 9, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 300, "exchanges.parquet": 4553 diff --git a/data/09-electronics/processes.parquet b/data/09-electronics/processes.parquet index 8fda56a..cd86dd1 100644 Binary files a/data/09-electronics/processes.parquet and b/data/09-electronics/processes.parquet differ diff --git a/data/10-building-services/metadata.json b/data/10-building-services/metadata.json index 898035b..e144636 100644 --- a/data/10-building-services/metadata.json +++ b/data/10-building-services/metadata.json @@ -3,7 +3,10 @@ "title": "Building services", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 10, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 268, "exchanges.parquet": 11887 diff --git a/data/10-building-services/processes.parquet b/data/10-building-services/processes.parquet index fc75788..ead548e 100644 Binary files a/data/10-building-services/processes.parquet and b/data/10-building-services/processes.parquet differ diff --git a/data/99-obsolete/metadata.json b/data/99-obsolete/metadata.json index 9d8828f..1aa82b3 100644 --- a/data/99-obsolete/metadata.json +++ b/data/99-obsolete/metadata.json @@ -3,7 +3,10 @@ "title": "Obsolete (BAFU legacy link targets)", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 99, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 779, "exchanges.parquet": 27492 diff --git a/data/99-obsolete/processes.parquet b/data/99-obsolete/processes.parquet index 508af30..ce328cb 100644 Binary files a/data/99-obsolete/processes.parquet and b/data/99-obsolete/processes.parquet differ diff --git a/data/README.md b/data/README.md index f2b7a31..3dbc357 100644 --- a/data/README.md +++ b/data/README.md @@ -15,10 +15,25 @@ data-type id (`agriculture`, `electricity`, `chemicals`) and `` sets folder ordering / resolution precedence (lower wins when records overlap). Inventory is organized by **sector, not by datasource**. Upstream sources -(BAFU, Agribalyse, …) each span many sectors, so the source is deliberately *not* -recorded in this repo — `sentier-importers` keeps the import log of which source -supplied which rows. This repo holds only the harmonized, sector-organized -result. +(BAFU, Agribalyse, …) each span many sectors, so a folder may hold rows from +several sources. Every process row is tagged with its `source` (and optional +`source_version`), and each folder's `metadata.json` lists the `sources` present, +so a consumer can pick folders by metadata and rows by column. Precedence between +folders is by rank; within a folder the consumer selects by `source` (there is no +intra-folder precedence rule). The import log of which upstream file produced +which rows stays in `sentier-importers`. + +### Source ids + +`source` is a lower-kebab datasource id, `-`, matching the +vocab Source IRI slug (`https://vocab.sentier.dev/sources/`) and the +sentier-mappings pair-folder prefix. The publisher's release goes in +`source_version`, never in the id, so a re-release does not rename the database a +consumer's project depends on. + +| datasource | `source` | `source_version` | +|---|---|---| +| BAFU:2026 v1 LCI database | `bafu-2026` | `v1` | > Background datasets that exist only as link targets (e.g. ecoinvent) are not a > sector; they are resolved through `sentier-mappings`, not stored here as their diff --git a/schema/exchange.yaml b/schema/exchange.yaml index cbffef0..53dad5b 100644 --- a/schema/exchange.yaml +++ b/schema/exchange.yaml @@ -1,5 +1,6 @@ # Schema for exchanges.parquet — one row per exchange (technosphere + biosphere # edges) of a process. Joined to processes.parquet on process_id. +# Provenance is on the parent process row (processes.source), joined on process_id. table: exchanges file: exchanges.parquet diff --git a/schema/metadata.schema.json b/schema/metadata.schema.json index eec3d3f..f76779a 100644 --- a/schema/metadata.schema.json +++ b/schema/metadata.schema.json @@ -2,19 +2,51 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://data.sentier.dev/inventory/schema/metadata", "title": "Sentier inventory sector metadata", - "description": "Provenance file (metadata.json) carried by each data/-/ folder. Inventory is organized by sector / data type only; which upstream source supplied which rows is logged by sentier-importers, not recorded here.", + "description": "Provenance file (metadata.json) carried by each data/-/ folder. Inventory is organised by sector; \"sources\" lists every datasource id present in this folder's processes.parquet \"source\" column. The source-to-row import log stays in sentier-importers.", "type": "object", - "required": ["sector", "title", "rank", "schema_version"], + "required": [ + "sector", + "title", + "rank", + "schema_version", + "sources" + ], "additionalProperties": true, "properties": { - "sector": { "type": "string", "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals." }, - "title": { "type": "string" }, - "description": { "type": "string" }, - "rank": { "type": "integer", "minimum": 1, "description": "Folder ordering / resolution precedence; lower wins." }, - "schema_version": { "type": "string" }, + "sector": { + "type": "string", + "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals." + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "rank": { + "type": "integer", + "minimum": 1, + "description": "Folder ordering / resolution precedence; lower wins." + }, + "schema_version": { + "type": "string" + }, + "sources": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9.]+)*$" + }, + "description": "Datasource ids present in this folder's processes.parquet source column, e.g. [\"bafu-2026\"]. Must equal the column's distinct values (CI-checked)." + }, "row_counts": { "type": "object", - "additionalProperties": { "type": "integer", "minimum": 0 } + "additionalProperties": { + "type": "integer", + "minimum": 0 + } } } } diff --git a/schema/process.yaml b/schema/process.yaml index 336d726..eb966c4 100644 --- a/schema/process.yaml +++ b/schema/process.yaml @@ -38,6 +38,19 @@ columns: type: enum enum: process_type # see common.yaml required: true + - name: source + type: string + required: true + description: > + Datasource id of the upstream dataset this row was imported from, lower-kebab + -, e.g. bafu-2026. Must appear in the sector's + metadata.json "sources" list. Consumers select rows by this column. + - name: source_version + type: string + required: false + description: > + Publisher's release/version of that dataset, e.g. v1. Free-form; never part of + the source id. - name: technology type: string required: false diff --git a/scripts/validate.py b/scripts/validate.py index d84b504..cae86dd 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -10,7 +10,9 @@ required columns present and non-null, column types match, enum columns only hold allowed values; - ``processes.process_id`` is unique per folder and every - ``exchanges.process_id`` resolves to a process in the same folder. + ``exchanges.process_id`` resolves to a process in the same folder; +- the distinct ``processes.source`` values equal ``metadata.json`` ``sources`` (both + directions), so consumers can select folders by metadata without opening parquet. Columns declared optional may be omitted entirely. A column whose values are all null passes the type check regardless of its physical type (an all-null @@ -120,6 +122,27 @@ def validate_folder(folder, process_cols, exchange_cols, enums, metadata_schema, f"with no process row (e.g. {sorted(orphans)[:3]})" ) + if processes is not None and "source" in processes.column_names and "sources" in meta: + validate_sources(folder, processes["source"], meta["sources"], errors) + + +def validate_sources(folder, source_column, declared, errors): + """``metadata.json`` ``sources`` must equal the distinct ``processes.source`` values.""" + seen = set(pc.drop_null(source_column.combine_chunks()).unique().to_pylist()) + declared = set(declared) + undeclared = seen - declared + if undeclared: + errors.append( + f"{folder.name}/processes.parquet: source values not declared in " + f"metadata.json sources: {sorted(undeclared)}" + ) + absent = declared - seen + if absent: + errors.append( + f"{folder.name}/metadata.json: sources declared but absent from " + f"processes.parquet: {sorted(absent)}" + ) + def main(): process_cols = load_table_schema("process.yaml")