feat(schema): structured source tag on processes (schema 0.2.0)

README.md

@@ -39,15 +39,15 @@ scripts/validate.py      # CI validator, self-contained
 
 ## Data
 
-Inventory is organized by sector, not by source. Which source supplied which rows is logged in `sentier-importers`.
+Inventory is organized by sector, not by source. Each process row carries a `source` tag (`bafu-2026`) and each folder's `metadata.json` lists the `sources` it holds, so consumers select by source without a source-named folder. See [data/README.md](data/README.md) for the id convention.
 
 Each `data/<NN>-<sector>/` folder holds:
 
 | file | rows |
 |---|---|
 | `processes.parquet` | one per process |
 | `exchanges.parquet` | one per exchange (technosphere and biosphere edges) |
-| `metadata.json` | sector, title, rank, schema_version, row_counts |
+| `metadata.json` | sector, title, rank, schema_version, sources, row_counts |
 
 - The `NN` prefix orders sectors. Lower wins when records overlap.
 - Sector ids are lower-kebab: `agriculture`, `electricity`, `chemicals`.

data/01-agriculture/metadata.json

@@ -3,7 +3,10 @@
   "title": "Agriculture",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 1,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 210,
     "exchanges.parquet": 7634

data/02-electricity/metadata.json

@@ -3,7 +3,10 @@
   "title": "Electricity",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 2,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 2033,
     "exchanges.parquet": 39133

data/03-chemicals/metadata.json

@@ -3,7 +3,10 @@
   "title": "Chemicals",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 3,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 510,
     "exchanges.parquet": 35566

data/04-transport/metadata.json

@@ -3,7 +3,10 @@
   "title": "Transport",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 4,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 2017,
     "exchanges.parquet": 91780

data/05-energy/metadata.json

@@ -3,7 +3,10 @@
   "title": "Energy carriers",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 5,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 3188,
     "exchanges.parquet": 46489

data/06-waste/metadata.json

@@ -3,7 +3,10 @@
   "title": "Waste management",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 6,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 848,
     "exchanges.parquet": 25777

data/07-construction/metadata.json

@@ -3,7 +3,10 @@
   "title": "Construction",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 7,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 927,
     "exchanges.parquet": 97366

data/08-materials/metadata.json

@@ -3,7 +3,10 @@
   "title": "Materials",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 8,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 867,
     "exchanges.parquet": 32386

data/09-electronics/metadata.json

@@ -3,7 +3,10 @@
   "title": "Electronics",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 9,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 300,
     "exchanges.parquet": 4553

data/10-building-services/metadata.json

@@ -3,7 +3,10 @@
   "title": "Building services",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 10,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 268,
     "exchanges.parquet": 11887

data/99-obsolete/metadata.json

@@ -3,7 +3,10 @@
   "title": "Obsolete (BAFU legacy link targets)",
   "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.",
   "rank": 99,
-  "schema_version": "0.1.0",
+  "schema_version": "0.2.0",
+  "sources": [
+    "bafu-2026"
+  ],
   "row_counts": {
     "processes.parquet": 779,
     "exchanges.parquet": 27492

data/README.md

@@ -15,10 +15,25 @@ data-type id (`agriculture`, `electricity`, `chemicals`) and `<NN>` sets folder
 ordering / resolution precedence (lower wins when records overlap).
 
 Inventory is organized by **sector, not by datasource**. Upstream sources
-(BAFU, Agribalyse, …) each span many sectors, so the source is deliberately *not*
-recorded in this repo — `sentier-importers` keeps the import log of which source
-supplied which rows. This repo holds only the harmonized, sector-organized
-result.
+(BAFU, Agribalyse, …) each span many sectors, so a folder may hold rows from
+several sources. Every process row is tagged with its `source` (and optional
+`source_version`), and each folder's `metadata.json` lists the `sources` present,
+so a consumer can pick folders by metadata and rows by column. Precedence between
+folders is by rank; within a folder the consumer selects by `source` (there is no
+intra-folder precedence rule). The import log of which upstream file produced
+which rows stays in `sentier-importers`.
+
+### Source ids
+
+`source` is a lower-kebab datasource id, `<publisher>-<edition>`, matching the
+vocab Source IRI slug (`https://vocab.sentier.dev/sources/<source>`) and the
+sentier-mappings pair-folder prefix. The publisher's release goes in
+`source_version`, never in the id, so a re-release does not rename the database a
+consumer's project depends on.
+
+| datasource | `source` | `source_version` |
+|---|---|---|
+| BAFU:2026 v1 LCI database | `bafu-2026` | `v1` |
 
 > Background datasets that exist only as link targets (e.g. ecoinvent) are not a
 > sector; they are resolved through `sentier-mappings`, not stored here as their

schema/exchange.yaml

@@ -1,5 +1,6 @@
 # Schema for exchanges.parquet — one row per exchange (technosphere + biosphere
 # edges) of a process. Joined to processes.parquet on process_id.
+# Provenance is on the parent process row (processes.source), joined on process_id.
 
 table: exchanges
 file: exchanges.parquet

schema/metadata.schema.json

@@ -2,19 +2,51 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://data.sentier.dev/inventory/schema/metadata",
   "title": "Sentier inventory sector metadata",
-  "description": "Provenance file (metadata.json) carried by each data/<NN>-<sector>/ folder. Inventory is organized by sector / data type only; which upstream source supplied which rows is logged by sentier-importers, not recorded here.",
+  "description": "Provenance file (metadata.json) carried by each data/<NN>-<sector>/ folder. Inventory is organised by sector; \"sources\" lists every datasource id present in this folder's processes.parquet \"source\" column. The source-to-row import log stays in sentier-importers.",
   "type": "object",
-  "required": ["sector", "title", "rank", "schema_version"],
+  "required": [
+    "sector",
+    "title",
+    "rank",
+    "schema_version",
+    "sources"
+  ],
   "additionalProperties": true,
   "properties": {
-    "sector": { "type": "string", "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals." },
-    "title": { "type": "string" },
-    "description": { "type": "string" },
-    "rank": { "type": "integer", "minimum": 1, "description": "Folder ordering / resolution precedence; lower wins." },
-    "schema_version": { "type": "string" },
+    "sector": {
+      "type": "string",
+      "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals."
+    },
+    "title": {
+      "type": "string"
+    },
+    "description": {
+      "type": "string"
+    },
+    "rank": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Folder ordering / resolution precedence; lower wins."
+    },
+    "schema_version": {
+      "type": "string"
+    },
+    "sources": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "type": "string",
+        "pattern": "^[a-z0-9]+(-[a-z0-9.]+)*$"
+      },
+      "description": "Datasource ids present in this folder's processes.parquet source column, e.g. [\"bafu-2026\"]. Must equal the column's distinct values (CI-checked)."
+    },
     "row_counts": {
       "type": "object",
-      "additionalProperties": { "type": "integer", "minimum": 0 }
+      "additionalProperties": {
+        "type": "integer",
+        "minimum": 0
+      }
     }
   }
 }

schema/process.yaml

@@ -38,6 +38,19 @@ columns:
     type: enum
     enum: process_type      # see common.yaml
     required: true
+  - name: source
+    type: string
+    required: true
+    description: >
+      Datasource id of the upstream dataset this row was imported from, lower-kebab
+      <publisher>-<edition>, e.g. bafu-2026. Must appear in the sector's
+      metadata.json "sources" list. Consumers select rows by this column.
+  - name: source_version
+    type: string
+    required: false
+    description: >
+      Publisher's release/version of that dataset, e.g. v1. Free-form; never part of
+      the source id.
   - name: technology
     type: string
     required: false

scripts/validate.py

@@ -10,7 +10,9 @@
   required columns present and non-null, column types match, enum columns only
   hold allowed values;
 - ``processes.process_id`` is unique per folder and every
-  ``exchanges.process_id`` resolves to a process in the same folder.
+  ``exchanges.process_id`` resolves to a process in the same folder;
+- the distinct ``processes.source`` values equal ``metadata.json`` ``sources`` (both
+  directions), so consumers can select folders by metadata without opening parquet.
 
 Columns declared optional may be omitted entirely. A column whose values are
 all null passes the type check regardless of its physical type (an all-null
@@ -120,6 +122,27 @@ def validate_folder(folder, process_cols, exchange_cols, enums, metadata_schema,
                     f"with no process row (e.g. {sorted(orphans)[:3]})"
                 )
 
+    if processes is not None and "source" in processes.column_names and "sources" in meta:
+        validate_sources(folder, processes["source"], meta["sources"], errors)
+
+
+def validate_sources(folder, source_column, declared, errors):
+    """``metadata.json`` ``sources`` must equal the distinct ``processes.source`` values."""
+    seen = set(pc.drop_null(source_column.combine_chunks()).unique().to_pylist())
+    declared = set(declared)
+    undeclared = seen - declared
+    if undeclared:
+        errors.append(
+            f"{folder.name}/processes.parquet: source values not declared in "
+            f"metadata.json sources: {sorted(undeclared)}"
+        )
+    absent = declared - seen
+    if absent:
+        errors.append(
+            f"{folder.name}/metadata.json: sources declared but absent from "
+            f"processes.parquet: {sorted(absent)}"
+        )
+
 
 def main():
     process_cols = load_table_schema("process.yaml")