From 2837e1b1ff1b8e2b7a8f3784c52c1d972aeb0e02 Mon Sep 17 00:00:00 2001 From: Laurenz Bougan Date: Thu, 1 Oct 2026 17:15:11 +0200 Subject: [PATCH] feat(schema): structured source tag on processes (schema 0.2.0) Rows were tagged with their upstream dataset only in free text (the ToU citation in `comment`), so a consumer could not select by source and sentier-brightway imported the whole inventory as "bafu-2026". This adds: - processes.parquet: `source` (required, datasource id, edition only, e.g. bafu-2026) and `source_version` (optional, e.g. v1), after process_type. Exchanges inherit through process_id. - metadata.json: required `sources` list; schema_version 0.2.0. - scripts/validate.py: `sources` must equal the distinct processes.source values, both directions, so consumers can select folders by metadata without opening parquet. - data/: all 11 sectors regenerated by sentier-importers (run_bafu_local.py at 732ba80) with source=bafu-2026, source_version=v1. Every pre-existing column and row is unchanged; exchanges.parquet files are byte-identical to main. - README, data/README.md: provenance rule and source-id convention. --- README.md | 4 +- data/01-agriculture/metadata.json | 5 +- data/01-agriculture/processes.parquet | Bin 21872 -> 22392 bytes data/02-electricity/metadata.json | 5 +- data/02-electricity/processes.parquet | Bin 114768 -> 115289 bytes data/03-chemicals/metadata.json | 5 +- data/03-chemicals/processes.parquet | Bin 63264 -> 63784 bytes data/04-transport/metadata.json | 5 +- data/04-transport/processes.parquet | Bin 215671 -> 216192 bytes data/05-energy/metadata.json | 5 +- data/05-energy/processes.parquet | Bin 217013 -> 217534 bytes data/06-waste/metadata.json | 5 +- data/06-waste/processes.parquet | Bin 78234 -> 78754 bytes data/07-construction/metadata.json | 5 +- data/07-construction/processes.parquet | Bin 90814 -> 91335 bytes data/08-materials/metadata.json | 5 +- data/08-materials/processes.parquet | Bin 88463 -> 88983 bytes data/09-electronics/metadata.json | 5 +- data/09-electronics/processes.parquet | Bin 36589 -> 37109 bytes data/10-building-services/metadata.json | 5 +- data/10-building-services/processes.parquet | Bin 25797 -> 26317 bytes data/99-obsolete/metadata.json | 5 +- data/99-obsolete/processes.parquet | Bin 84828 -> 85348 bytes data/README.md | 23 ++++++++-- schema/exchange.yaml | 1 + schema/metadata.schema.json | 48 ++++++++++++++++---- schema/process.yaml | 13 ++++++ scripts/validate.py | 25 +++++++++- 28 files changed, 143 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 5d05044..c46be0b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ scripts/validate.py # CI validator, self-contained ## Data -Inventory is organized by sector, not by source. Which source supplied which rows is logged in `sentier-importers`. +Inventory is organized by sector, not by source. Each process row carries a `source` tag (`bafu-2026`) and each folder's `metadata.json` lists the `sources` it holds, so consumers select by source without a source-named folder. See [data/README.md](data/README.md) for the id convention. Each `data/-/` folder holds: @@ -47,7 +47,7 @@ Each `data/-/` folder holds: |---|---| | `processes.parquet` | one per process | | `exchanges.parquet` | one per exchange (technosphere and biosphere edges) | -| `metadata.json` | sector, title, rank, schema_version, row_counts | +| `metadata.json` | sector, title, rank, schema_version, sources, row_counts | - The `NN` prefix orders sectors. Lower wins when records overlap. - Sector ids are lower-kebab: `agriculture`, `electricity`, `chemicals`. diff --git a/data/01-agriculture/metadata.json b/data/01-agriculture/metadata.json index ae24205..b73e4b8 100644 --- a/data/01-agriculture/metadata.json +++ b/data/01-agriculture/metadata.json @@ -3,7 +3,10 @@ "title": "Agriculture", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 1, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 210, "exchanges.parquet": 7634 diff --git a/data/01-agriculture/processes.parquet b/data/01-agriculture/processes.parquet index 6637559d77421465bb69ef8535b5dc3ebff7dd57..3a7bce410e61cc9c710f5d9586fea06698e329ee 100644 GIT binary patch delta 688 zcmeycit)!f#tnjY&QhXsKB7#b3_=VHyat>M3=Bz$X{EYG21aJ045DJ9QaYkbm_-Fd z*+ki7%osE{;j$74=H&A>qIx`{0x)yfM3{i)lo?_(fT_$-0*FCcHVfM6@$sH9WsqPi zPR>ZpO%xTI?BlaWiLE%lv?w`MmBB{_XpAV2Dw6~sL@2&2wWv5VKX39I9}C7kn^k@N zn7yE?MVZ8wFpI5V6x+b4cI^kF+MFMZINd8K$RMdB$^vvA69bSC08%z$M`R~628cPK z*#t4RPOObl?Z;0>wKG4#rb0b|)tnQP0>md52Z*y>na8Ad>DS~50Ws`-YnargEn?bS z=cU0oSvp8TZQXZfpmWtY#FlMlVvv-P;BgEJ@(;Iy1cSsGHhmBZpO%xTH{M={F^Yvq%{54RV zecl`<^{|8OganG$DM^+BLw^5b9!i)=?nN9SV4Ku5=9M^{HjmvkqffGdb!0c1FO0U3@? z=^&;rhyY1AI(zDuO`aVl>)`uNG&@a>1FAgx**DnVt4FxH71S#yC diff --git a/data/02-electricity/metadata.json b/data/02-electricity/metadata.json index 21420ac..ceb9eaf 100644 --- a/data/02-electricity/metadata.json +++ b/data/02-electricity/metadata.json @@ -3,7 +3,10 @@ "title": "Electricity", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 2, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 2033, "exchanges.parquet": 39133 diff --git a/data/02-electricity/processes.parquet b/data/02-electricity/processes.parquet index 95d41c587bcea5660206c71374f74ed48f895e0c..27bf57cca529bfdbae2251340ae5f59bbbcfeb11 100644 GIT binary patch delta 687 zcmcc6z<#rZyg9KY~az<)yqNv#Po7))ID6tjimlh?bsxtV<08J6)QDu_gg9yc! zr4|)u=I2eX+|FphxM%yO?Tj;-jiKsAnZzE+i>+W3+rX%{VLy-Bw|zV~Jutm+7o!M^ z*r&Nn(>Ls76xT*I9%53RSR13-r2{-_%MO4Ig1Q^5m09eFJxHh6bno4a5^TFB@T#pj zI6ZSWV?Nut1H5VzHf+DQo6(k0ZOs#Lprh0{#17o$WssDS;BgEJ@(;Iy_*dc#n?48_ zPj}qQ=ultm2n4>4>5h&Dj;@Z5Ug=Jbj!xN*fj}mR9{^-Hx`5O;=70!y5aC!3B7h2k zQZCy1`W8Tjs~b=?P!MPkkOLwhAQ?zGWhEIImPCb{8aO&t0o7YZ>X+&3gCv}$7wlsc zll2CvO>qy)N_00ZOLDi&O9fgR9AK`mZxW(C{qsIXWsj;PAFtZ2wc8W-GTJh#wLKCSdnB*MA=Y)9 zmqAiSg2yo^$Uoc)VyeU$R(%ktn0{m*qk~1Zqobp9v16d4W3r>GqoYf@6Hvev#IFD{ z9KC=HN2hcU(-%a5BpjVR^~b+#6@fK^ctGtS<2-_LB8@UilZ-0LqKvb*D;{9vWn!E&UH=fHG$%tnCj&!(V~`;L DI|W*4 diff --git a/data/03-chemicals/metadata.json b/data/03-chemicals/metadata.json index c19214e..4bfbb30 100644 --- a/data/03-chemicals/metadata.json +++ b/data/03-chemicals/metadata.json @@ -3,7 +3,10 @@ "title": "Chemicals", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 3, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 510, "exchanges.parquet": 35566 diff --git a/data/03-chemicals/processes.parquet b/data/03-chemicals/processes.parquet index 5eb3aae7c562d4faab632f960312d8824d6ecba5..9c1b2bcf49cf827740dc88dda0078135175d1283 100644 GIT binary patch delta 718 zcmZ4Rj(No=<_+DG>!n2Hd_$YQSH<@7PSfISnxSsQU~Z>22mykARz#x zY{c%kOR`C-0eu6sQ;k7Jlt}nI{ux~!|LX2_p>CXykSMKuy-LJ+WHf23KgQScE zk7H1ff4CJSTqMr0>4SjrWWg^E^~H`r;Om&~=xE^R>gec|?&Rp`lg$qlPaHp(eW0@{-bQehrKzyXmUd!YJ5H^2MB%fz^6^8arVlli`LaRl))Fa$UT G83F*Hl)&Tw delta 280 zcmZ4SiFv_0<_+DGH>*uq9nQPPltF^6I5{IVH&Ik*^1atKk`lNCQo zvi)dbSDSEdv%|*^OuSzfv8&yh$u48YpwYoNS>d_#=7=w%jB4xd@{0XoSK|=7w}zcT zQbvNuF(}AC+zMi<#2Hq75U7|u`KyCPwxgq?bFpKfqhqq8tD~b!x)V^q6~wOqG90~t z3`eJQ5Yrb#fFvB9J@v~bd;gI2@BoQK8d;Vkg@=`#K5B8@UilZ-0LqKvaQfB44B#5iX%<4*Pf diff --git a/data/04-transport/metadata.json b/data/04-transport/metadata.json index 120ad40..84035eb 100644 --- a/data/04-transport/metadata.json +++ b/data/04-transport/metadata.json @@ -3,7 +3,10 @@ "title": "Transport", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 4, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 2017, "exchanges.parquet": 91780 diff --git a/data/04-transport/processes.parquet b/data/04-transport/processes.parquet index d5e12e52ffd2f5327e99d95a8e504df9d07ac8d1..6ca7c84c10ffd3245c429d871db90a5cf819c3a3 100644 GIT binary patch delta 742 zcmex9g|}fQZ$k^?7A7f)dMQykA5kVz1|bFpUIR`B28N`>v{GFo10yq022n9lDIL*6 z@}dHwY@%#3W(*pfa9IfiQ&147ja!2mNFI`BWIQC#Aj%@jBPswhmraBTC{<>N%|Ldh zGD8U`TK={gR~7L0qgr#dj}F&jhGi!zBFk{4USD7Jx7?ZpEDwRH~!aC&0; zLq}#&=1a4grWZOgi|L>l4>74utc_8v>yd!ky@z0fpl%0iWooep>zqE>nOTy3)+$N0 zCyxZSvuH6NW1L>;%52Ym>6xV3vYV3I6)c&P8K*ySWEL{H^G8{dO;QaQYCvC!otP60 zbgmkQ*s=yG21ywS9><^{|8OfvNJyMv(+2_L=^5_K4)w*3K;Y|`?&xUX=<4X`mG0!| z=#=dk2xNlz0YHYM3rLM)4v26E5su{`0;muu<)W>xZvkYux&c)K1%Ur97FHJ*tw7!zv?|l`Hl>1;n1>K>LL|s0sQ%FHt{%+1Oxk;Z R(T^W7B=RsY1ULp60syavxgU=F=dcoD^AWx%}o>)ny&7^yk$vx3DA z$*XaQefTRm{h|l6FgvS02vkge?ZNC&pY7=A=v?d==;)a2=<4X`lI{c)a0T%zfDA`3 zAj8oq9mMnn5g-XiXHWeyeSILq$tegV<`@YgKpLUI14Ks}S(YS)hn1!{IvIe}7YCT@ z>z4y10z*NDID!?)fei$4fyRQY@d(O^G|DJVGO8$xGS1%a;l<3$#5iYqybrSkj|>9? N!)GoAh5*MPLjeEdTRZ>& diff --git a/data/05-energy/metadata.json b/data/05-energy/metadata.json index 2e1ed34..137c445 100644 --- a/data/05-energy/metadata.json +++ b/data/05-energy/metadata.json @@ -3,7 +3,10 @@ "title": "Energy carriers", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 5, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 3188, "exchanges.parquet": 46489 diff --git a/data/05-energy/processes.parquet b/data/05-energy/processes.parquet index 0922e7d1eecb47a04c7aaa2c4384d9c436e52f5b..0725a8b5fcc6632f23e1b957075fcfe7e19dd09e 100644 GIT binary patch delta 680 zcmdlwpLgG4-i8*&ElgWpOG$~!`G_)!G6*p+@EUM3Ffb$~rj_a%85o&OZ+OilBFYKp zOCXrj=f7qW73L8YfN5nDVFGF`GX(2lXDTz40OIXHRa2sP&zLeuuoWj~q~<1yicNQL zV_u`gR-9j2l$@%{;3EUnCd#AAB*6y}iZ4qoD$dN$oBqI!*@AJ;c1d^U70ljH^`cB- zFAT+2Fp6zpRGYJ2T` zjz~_ouwwo>J;8@rL2cD5L!h(OIK=v9OEE~wNbooY1^I_tK|)0044Xa(7*Aj3!|YIB z><9$Dj_Hn$29B*j)6cXh#vrCIJ$t;IOc!|cM#!N4kCaGfl@Bo`uY|? zhN~M;HBb;}5Rd~RARrk?Ib|go8J0wan;JMeRRPsoM(UU8>w_emreF1C7L)Y`sZDVY z%Sv=NElYB@%u5AY8ysM+uWu5fJw3vYS=pm1$vCVs(kLt~B{!@x*(j$h320L)NQrp} q0VhO)Y=Y_!-QMiW%*({MXZl2cX6fk*{F&J}8D4WSFa$UT83F(?47ynW delta 287 zcmdnDn0M=Z-i8*&ElgWpZ})%0R3F8=#*{&VtvER&H8)XIX!>n8<~7r|x-;uA&e{IR zop}ZGbQd3HNw!yQl4_6EO;7e=ZeTm}LQ<{m(Ds);%uke%bVg0nD-<9w4DeBg>Mc@UYSpM<)aQGJXBx0CRo)a**0kkZMP; zBCtjf52zhvoJUYjq)|p`l2JujlyUaOri`z3=F&moD2*MNr`Etx<&>@W}*zDVxm$yq6>IM z1w`3I*<{QZG&te15(uWCAW$2(1~ZUcz{|+EfR{m(MU+QW0A?`Y~b z5G#D)y_iVS-V7$bv4^=P9B({K8Yz3p(21d0f8#vTfZQ#J_f$4U7jFLxU=-6qGZ|t?omd;ATH7WLwObp(WZ4DTeJ*tw7!zv?|l`R;PlLn1>K> hQzXbTsQ%FH8TyR8Oxk;ZF^nHEtmbB52yhHC1OShat&9Kw delta 308 zcmZ4VoMqNymWC~i5=^R6*vkYFoL&PdHo6cw8OU!8Hy^qU%tI*fC+vuQG3 zVxB%xmr;VP=PQTWstwcE>oWdi>zlx-HlbzvIz2{9MzuAsgvA!{s&R-dn9Io^DI>w- z7!>3mZUr$<;tZ=k2vkhxHDGk8&vtZlbS`!bbaYI1baixeNp}JYxPtfOri`z3=F&moD2*MNr`Etx<&>@X44hK7)3-m;d}`M zbGp44qo^>Cr~phWn+Ow7YndTf4?9ztp#%_b2dWZr<~?J|Ai-9goRONFC@MBRA(C;8 z5?gV8X;E^jDua&WYx7=VNTkg^f`BLg;Q`r%kcNw$W~TxvINPk$847{PY!I+xm< zBikL~7(E%)t~5&kU8u$(cJ4VBgQScEk7H1ff4CLI?-FO&^g+OQ`uccAhx%eiAnPBr4q0z|pA+sNOPCzf4~rB;hpuUIL?-tT#w)ihEdAqPuBX zlDlPID$v^C0CRnPlMwCciHVHL9#u)kVU>|aVQDG3VU@{7Ib}&en^Hka%tHt`ArfR0 mRDbC9o&-i-CdNI}7bG!?PG6tI$OaJQ=~tr|br|PtXNqR* zVxI06&nU@uY6F+rqMOrG;~68^7hL61dv=g(``=JT4aVtj5*QWK)-_6q?c-D95S#Io zi$PLGg2yo^$Uoc)VyeU$R(%ktm~NEF=wOlU=;-KN>=@|inC$54=;)H}1Qc)u@hgA~ zM=v15(J39o^aT+h2}fs7{j%w=k{M+^JU~K`MwTT>;bEmIj!p*pW%~NX0p|MpB%_M5DC6wy{Yi|xOpJ4;FGyjO=443YWMBwz3^D`& DnMhe- diff --git a/data/08-materials/metadata.json b/data/08-materials/metadata.json index 2bdef7b..11cb64a 100644 --- a/data/08-materials/metadata.json +++ b/data/08-materials/metadata.json @@ -3,7 +3,10 @@ "title": "Materials", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 8, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 867, "exchanges.parquet": 32386 diff --git a/data/08-materials/processes.parquet b/data/08-materials/processes.parquet index 1beba7a1087d2849c452f4d089c75a844637ea8b..250a2e970e085632d4e914dfa359c3106212c0dc 100644 GIT binary patch delta 699 zcmeC*%sPEL>xSuH>!n2Hd__kpUVa%A?98!3PnFFH0>d z&dkr7?iavl!MJC8djR7eW@D&&Q6{ltykaXD#WpaiJ-WuOw&EH)PWMlr7{n;b^sRFG z#b8D;9W>J+2GxnRF{-uPU{||w9c&KN)exOM(qNs_okJKU*|&6asolH5K0VNhF=zVu z5Jm^~6C1hI{>|aqe%qT-g>kx77^8yPms`R>=c#dsy*j|fASolk;}{g=A8rNlxx^VZ zeGo97-VnyJQx>8_vkf#JFdAVI-sI^oB@AHW0zi5ys8H5a1YO2moL? ByaE6K delta 289 zcmbQfowa{6>xSuHHyeH9isD^k${@j3oSc!Gn2I7EbEX@IFbYmj3S-n~o3x%w?Zm9@)592D8P(3+5EeVetHvR= zWgi!Vq>KcQV^ENPxD~``i8HMFAW$)#DT2|#BHPi?(Ye?$(9tp3(bdt>CEW=q;0oeb z02z*6K!&4JI*92DB0v(3&Yt>Z(|1KN%6fQ!gd&YBOOnFFN>d!24D`$N^@{_{_4Ug^ zYC}P)9l?sg8bLguc93x%K{=5|8Kp@^6=hMz+1ukI8F`r)=S(k%W|Zb+c*)7Y5a1YO F2mn5dSh4^B diff --git a/data/09-electronics/metadata.json b/data/09-electronics/metadata.json index e673647..9b2a1c4 100644 --- a/data/09-electronics/metadata.json +++ b/data/09-electronics/metadata.json @@ -3,7 +3,10 @@ "title": "Electronics", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 9, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 300, "exchanges.parquet": 4553 diff --git a/data/09-electronics/processes.parquet b/data/09-electronics/processes.parquet index 8fda56a1706887c19909fa6bb76e57aa84fd2c26..cd86dd1d00f7ccc5d731abfd9b5236a07380d249 100644 GIT binary patch delta 689 zcmaDmm+9+5rVT%$>ZL^Gd_Oc^BDijy-^a}!0yCf{jUqr_I6Us{x$s>u5mlsLnt4+6%M?{z!W z7drxhuVcESqk*HVqoY^4lcS?kwqqcW3E~F;8ICR>HI6wT!W~37mV*eOLZFn3w!Xdv zkm2eER1Fja8U*Bk2na|9QchV(MusI(;id+TPE|nlmXZ2p`uZRVC*vTnGLR~Wfnctq zH;77c56en)H!VwYx6Dfgx+6HiTwmWLM0@gtUS*G}B;&BkNTaZ{l-#h&WTTw2B%sZy sAQk2z1e_EJvJ0v|bo1^WUM9vplh5~wPQKU22E^DQrVT%$HcyVeuF1Q`ltF^6I5{IVH&Ik*a(U~T$$@P;jB_^EwM}83e7Z}L zecw-JwO6m1CiB#9o4l~wp6yE)i`u!y&Cj|+7}Y-AVi&u?qQ)UMVKNJYq>KcQV^ENP zxD~`Wi8HMFAW$(mq1VA8+tJa{x!5t#(J|T4)zQ%<-3ch*3gTA)8IE2+hNDwDi0KO= zKoXA5p892zB__yvcz{GAjVw!&!ox~a9GwjG%k=e&1I+dH%Rx#*L5dy0iop6nJfLon oX&ym2kwzJ%Nk$cAQO4Pu_xABJG0vHMae@qoBo_lifMbv$0KaTn<^TWy diff --git a/data/10-building-services/metadata.json b/data/10-building-services/metadata.json index 898035b..e144636 100644 --- a/data/10-building-services/metadata.json +++ b/data/10-building-services/metadata.json @@ -3,7 +3,10 @@ "title": "Building services", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 10, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 268, "exchanges.parquet": 11887 diff --git a/data/10-building-services/processes.parquet b/data/10-building-services/processes.parquet index fc75788139980e479074fc244cc86639c387cfc8..ead548e56ecf1888233f5a4a42a62d8864a428f5 100644 GIT binary patch delta 671 zcmX?llJV?W#tmmZ>!n2Hd_2t< z)nJGzbz*IdY9CfGsU2AXHU#Qku(k~xKy6}^Z^nzWAG*h+c4{TlWQlP7$!ZDqY|~yd zsdYWxoSmS`sJ3DOE6_b^9AYm%GBHTXNbooY1^I_tLHsFkhD{#?j3;kPbf_=$kPM`pvXYDpOQOO}4IG`Sfa)zH^~?12K@v`rpC*aPdV`dvxQAsWx|^0I zxm)I?0&NWrFxS^N3DKUMk*w@dm1G=N8EF)jmXaG*nQW9(mISmX6{NyEgn$DgLH0oP fhi;yn#LL9EXY#@n>B$>X*f<${xfmD%9D@u2pOCbw delta 271 zcmX?mmhtFG#tmmZH)nVW3-PWoWsqPiPR>ZpO%xTH+!3>8a#E}g zSoJ}mVzNcDgGIKZqoZ@NW1ypBvZJe`qf5FIP{0+$uK+R}y?_iyr*sh07es(09GyM& z%O-zFmG$rdi9{M%mL!FTm8Lj48R(bk>lX)@>+6?;l!k&7JAxH~^?`Um-5}FEf^s5_ mGD?$-D$1gavo}vo;bmf+GkJZQ=;VEA>>Lj`7#IQ^gA4(%?N@UE diff --git a/data/99-obsolete/metadata.json b/data/99-obsolete/metadata.json index 9d8828f..1aa82b3 100644 --- a/data/99-obsolete/metadata.json +++ b/data/99-obsolete/metadata.json @@ -3,7 +3,10 @@ "title": "Obsolete (BAFU legacy link targets)", "description": "Source: Life Cycle Inventory database of the Swiss Federal Administration, BAFU:2026.", "rank": 99, - "schema_version": "0.1.0", + "schema_version": "0.2.0", + "sources": [ + "bafu-2026" + ], "row_counts": { "processes.parquet": 779, "exchanges.parquet": 27492 diff --git a/data/99-obsolete/processes.parquet b/data/99-obsolete/processes.parquet index 508af3000bbf1c549db4da7a6827f35bf9550e4d..ce328cb5f04260f9c63801b00c60fa65ed632635 100644 GIT binary patch delta 706 zcmcaJjrGYa)(yR%oTWtNd_9Q<&wt`V?1EboxQ|xL}PO)Qk@-z=dNta`}l5CP{4AXdkHmEVki1J99 zGO!6qqFV$pvrepyQSHNNcC{m?!A3$oF`dzsQG{7+jl}fhu8jKZS9Wo#ojSulT~UwG ze!7htqXYY&Z=7mdPH}EGwPkE!oX+mSsG!!cLKNsqH4d?P9b62OG7>zFK|%iER*(RY zIK!q70>;yQJs2J8iyeW$*D>AE(ZJEw(a|g2${Z$=poPi_W=0LLIh E0AaAW$N&HU delta 283 zcmaDdi}lVl)(yR%Hfwx-=f}IoltF^6I5{IVH&Ik*y0R1Fn(2R?7)aU|*pGeTRQq?FbNdBb#umouot}&eYPXh(icRBD;}Cn%%Ecfl zBf;Ys6yzUn1u<3P468l}R7^kb$>?B_?da&}Tgec_?gSKY1@S9@3`Z{@ z!_g@n#PkIbAPGljPyMp#?LLgM9v&c}NF&RVr0}rP6h|im{W5+1;sA4f{c@1nP>^az zup+QV5D%ywWSmD(PNY#rX_8SzS(I`1b{%g|t1j diff --git a/data/README.md b/data/README.md index f2b7a31..3dbc357 100644 --- a/data/README.md +++ b/data/README.md @@ -15,10 +15,25 @@ data-type id (`agriculture`, `electricity`, `chemicals`) and `` sets folder ordering / resolution precedence (lower wins when records overlap). Inventory is organized by **sector, not by datasource**. Upstream sources -(BAFU, Agribalyse, …) each span many sectors, so the source is deliberately *not* -recorded in this repo — `sentier-importers` keeps the import log of which source -supplied which rows. This repo holds only the harmonized, sector-organized -result. +(BAFU, Agribalyse, …) each span many sectors, so a folder may hold rows from +several sources. Every process row is tagged with its `source` (and optional +`source_version`), and each folder's `metadata.json` lists the `sources` present, +so a consumer can pick folders by metadata and rows by column. Precedence between +folders is by rank; within a folder the consumer selects by `source` (there is no +intra-folder precedence rule). The import log of which upstream file produced +which rows stays in `sentier-importers`. + +### Source ids + +`source` is a lower-kebab datasource id, `-`, matching the +vocab Source IRI slug (`https://vocab.sentier.dev/sources/`) and the +sentier-mappings pair-folder prefix. The publisher's release goes in +`source_version`, never in the id, so a re-release does not rename the database a +consumer's project depends on. + +| datasource | `source` | `source_version` | +|---|---|---| +| BAFU:2026 v1 LCI database | `bafu-2026` | `v1` | > Background datasets that exist only as link targets (e.g. ecoinvent) are not a > sector; they are resolved through `sentier-mappings`, not stored here as their diff --git a/schema/exchange.yaml b/schema/exchange.yaml index cbffef0..53dad5b 100644 --- a/schema/exchange.yaml +++ b/schema/exchange.yaml @@ -1,5 +1,6 @@ # Schema for exchanges.parquet — one row per exchange (technosphere + biosphere # edges) of a process. Joined to processes.parquet on process_id. +# Provenance is on the parent process row (processes.source), joined on process_id. table: exchanges file: exchanges.parquet diff --git a/schema/metadata.schema.json b/schema/metadata.schema.json index eec3d3f..f76779a 100644 --- a/schema/metadata.schema.json +++ b/schema/metadata.schema.json @@ -2,19 +2,51 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://data.sentier.dev/inventory/schema/metadata", "title": "Sentier inventory sector metadata", - "description": "Provenance file (metadata.json) carried by each data/-/ folder. Inventory is organized by sector / data type only; which upstream source supplied which rows is logged by sentier-importers, not recorded here.", + "description": "Provenance file (metadata.json) carried by each data/-/ folder. Inventory is organised by sector; \"sources\" lists every datasource id present in this folder's processes.parquet \"source\" column. The source-to-row import log stays in sentier-importers.", "type": "object", - "required": ["sector", "title", "rank", "schema_version"], + "required": [ + "sector", + "title", + "rank", + "schema_version", + "sources" + ], "additionalProperties": true, "properties": { - "sector": { "type": "string", "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals." }, - "title": { "type": "string" }, - "description": { "type": "string" }, - "rank": { "type": "integer", "minimum": 1, "description": "Folder ordering / resolution precedence; lower wins." }, - "schema_version": { "type": "string" }, + "sector": { + "type": "string", + "description": "Lower-kebab sector / data-type id, e.g. agriculture, electricity, chemicals." + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "rank": { + "type": "integer", + "minimum": 1, + "description": "Folder ordering / resolution precedence; lower wins." + }, + "schema_version": { + "type": "string" + }, + "sources": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9.]+)*$" + }, + "description": "Datasource ids present in this folder's processes.parquet source column, e.g. [\"bafu-2026\"]. Must equal the column's distinct values (CI-checked)." + }, "row_counts": { "type": "object", - "additionalProperties": { "type": "integer", "minimum": 0 } + "additionalProperties": { + "type": "integer", + "minimum": 0 + } } } } diff --git a/schema/process.yaml b/schema/process.yaml index 336d726..eb966c4 100644 --- a/schema/process.yaml +++ b/schema/process.yaml @@ -38,6 +38,19 @@ columns: type: enum enum: process_type # see common.yaml required: true + - name: source + type: string + required: true + description: > + Datasource id of the upstream dataset this row was imported from, lower-kebab + -, e.g. bafu-2026. Must appear in the sector's + metadata.json "sources" list. Consumers select rows by this column. + - name: source_version + type: string + required: false + description: > + Publisher's release/version of that dataset, e.g. v1. Free-form; never part of + the source id. - name: technology type: string required: false diff --git a/scripts/validate.py b/scripts/validate.py index d84b504..cae86dd 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -10,7 +10,9 @@ required columns present and non-null, column types match, enum columns only hold allowed values; - ``processes.process_id`` is unique per folder and every - ``exchanges.process_id`` resolves to a process in the same folder. + ``exchanges.process_id`` resolves to a process in the same folder; +- the distinct ``processes.source`` values equal ``metadata.json`` ``sources`` (both + directions), so consumers can select folders by metadata without opening parquet. Columns declared optional may be omitted entirely. A column whose values are all null passes the type check regardless of its physical type (an all-null @@ -120,6 +122,27 @@ def validate_folder(folder, process_cols, exchange_cols, enums, metadata_schema, f"with no process row (e.g. {sorted(orphans)[:3]})" ) + if processes is not None and "source" in processes.column_names and "sources" in meta: + validate_sources(folder, processes["source"], meta["sources"], errors) + + +def validate_sources(folder, source_column, declared, errors): + """``metadata.json`` ``sources`` must equal the distinct ``processes.source`` values.""" + seen = set(pc.drop_null(source_column.combine_chunks()).unique().to_pylist()) + declared = set(declared) + undeclared = seen - declared + if undeclared: + errors.append( + f"{folder.name}/processes.parquet: source values not declared in " + f"metadata.json sources: {sorted(undeclared)}" + ) + absent = declared - seen + if absent: + errors.append( + f"{folder.name}/metadata.json: sources declared but absent from " + f"processes.parquet: {sorted(absent)}" + ) + def main(): process_cols = load_table_schema("process.yaml")