diff --git a/mysite/dpp/lca.py b/mysite/dpp/lca.py index 7fa28f4..8dcf92d 100644 --- a/mysite/dpp/lca.py +++ b/mysite/dpp/lca.py @@ -1,10 +1,20 @@ """Toolbox with functions for doing LCA calculations, and to import and export DPP data to Brightway. """ -import bw2data as bwd -import bw2io as bwi import datetime +import json import logging +import time +import requests + +from pathlib import Path +import pandas as pd +from django.conf import settings + +import bw2data as bwd +import bw2io as bwi +from sentier_brightway import import_db + logger = logging.getLogger(__name__) @@ -25,15 +35,121 @@ "human toxicity: non-carcinogenic, inorganics', 'comparative toxic unit for human (CTUh)", "human toxicity: non-carcinogenic, organics', 'comparative toxic unit for human (CTUh)')", } -DEFAULT_REMOTE_PROJECT = "ecoinvent-3.12-biosphere" +DPP_PROJECT = "L4M-DPP" +BIOSPHERE = "ecoinvent-3.12-biosphere" ECOINVENT = "ecoinvent-3-12-cutoff" +BONSAI_DB = "BONSAI v2.4.0" +BAFU_DB = "bafu-2026" + +CACHE_DIR = Path.cwd() / "_cache" +BONSAI_EXPORT_DIR = Path.cwd() / "bonsai_data" +CACHE_DIR.mkdir(exist_ok=True) +BONSAI_EXPORT_DIR.mkdir(parents=True, exist_ok=True) + +# --------------------------------------------------------------------------- +# Project setup and database import +# --------------------------------------------------------------------------- + +country_data = pd.read_csv(settings.DATA_DIR / "country_data.csv", index_col="country") -def setup_project(project_name: str) -> None: - """Initialize the project if needed, and check that it is complete.""" +def project_status(project_name: str) -> dict[str, bool]: + """Check if all databases in a project are fully processed. + + A database counts as True only if its processed file exists on disk + (an import that failed half-way leaves only a registered DB). + Returns {db_name: processed}, or {} if the project does not exist. + """ if project_name not in bwd.projects: - logger.debug("Creating a new Brightway project setup.") - bwi.remote.install_project(DEFAULT_REMOTE_PROJECT, project_name) + return {} bwd.projects.set_current(project_name) + return { + name: Path(bwd.Database(name).filepath_processed()).is_file() + for name in bwd.databases + } + +def staged_bonsai_export(export_dir: Path|None=None) -> tuple[Path, dict]: + """ + The export staged in the cache with its units fixed, plus the BONSAI -> biosphere map. + + v2.4.0 labels standing wood "standard cubic meter", ecoinvent "cubic meter", and the + importer's unit library does not know the former. For solid wood the two are the same + volume, so the label is aligned 1:1. The export itself is not modified. + """ + export_dir = export_dir or BONSAI_EXPORT_DIR + export_files = [ + "io_metadata.json", "product_value_table.gzip", + "technosphere_value_table.gzip", "extensions_value_table.gzip", + "index_table_technosphere.gzip", "index_table_extensions.gzip", + ] + stage = CACHE_DIR / "bonsai_export" + stage.mkdir(exist_ok=True) + for name in export_files: + link = stage / name + link.unlink(missing_ok=True) + if name != "io_metadata.json": + link.symlink_to((export_dir / name).resolve()) + + metadata = json.loads((export_dir / "io_metadata.json").read_text(encoding="utf-8")) + biosphere = bwd.Database(bwd.config.biosphere) + to_biosphere = {} + for code, flow in metadata.items(): + if "compartment" not in flow or not flow.get("source_id"): + continue # unlinked flows go to " biosphere" + target = biosphere.get(code=flow["source_id"]) + to_biosphere[code] = target["code"] + if flow.get("unit") == "standard cubic meter" and target["unit"] == "cubic meter": + print(f" unit aligned 1:1 to 'cubic meter': {flow['name']}") + flow["unit"] = "cubic meter" + (stage / "io_metadata.json").write_text(json.dumps(metadata), encoding="utf-8") + return stage, to_biosphere + +class BonsaiImporter(): + """ + Replacement for `from bw2io.importers.bonsai import BonsaiImporter`, + which requires a fork of bw2io. For now, we stick to core bw2io. + """ + def __init__(self, *args, **kwargs): + raise NotImplementedError() + +def import_bonsai() -> None: + """Import the BONSAI database using a staged export approach.""" + stage, to_biosphere = staged_bonsai_export() + importer = BonsaiImporter(stage, BONSAI_DB, b3mapping=to_biosphere) + importer.apply_strategies() + importer.write_database() + +def setup_project(project_name: str=DPP_PROJECT, background="BAFU") -> None: + """Initialize the project if needed. Add biosphere and background DB + and check the completeness. + """ + status = project_status(project_name) + + if project_name not in bwd.projects: + logger.debug("Creating a new Brightway project setup.") + if background=="BAFU" and (BAFU_DB not in status or not status[BAFU_DB]): + import_db(project=project_name, overwrite=False) + elif BIOSPHERE not in status or not status[BIOSPHERE]: + scratch = f"{project_name}_setup" + bwd.projects.set_current(scratch) + bwi.remote.install_project(project_key=BIOSPHERE, + project_name=project_name, overwrite_existing=True) + bwd.projects.set_current(project_name) + bwd.projects.delete_project(scratch, delete_dir=True) + else: + bwd.projects.set_current(project_name) + if background=="BONSAI" and (BONSAI_DB not in status or not status[BONSAI_DB]): + started = time.perf_counter() + logger.info(f" importing {BONSAI_DB} from {BONSAI_EXPORT_DIR} …") + if BONSAI_DB in bwd.databases: # left behind by an import that failed half-way + logger.info(f" removing an incomplete '{BONSAI_DB}'") + del bwd.databases[BONSAI_DB] + import_bonsai() + logger.info(f" done in {time.perf_counter() - started:,.0f} s") + + # Check and log the new status + for db, present in project_status(project_name).items(): + logger.info(f"{db:<10} {'present' if present else 'MISSING'}") + def ensure_methods(family: str): """ @@ -89,10 +205,47 @@ def prompt_choice(title: str, options: list, default_index: int = 0): print("Invalid choice, using default.") return options[default_index] + +# --------------------------------------------------------------------------- +# Conversion and communication functions +# --------------------------------------------------------------------------- + +def bonsai_api(endpoint: str) -> list: + """Pages of BONSAI API endpoint (no account needed), cached in CACHE_DIR. + Examples of `endpoint`: 'products/', 'activities/', 'product-corr/' + """ + # Load data from cache if available + path = CACHE_DIR / f"bonsai_api_{endpoint.strip('/')}.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + + rows = [] + url = "https://lca.aau.dk/api/" + endpoint + while url: + page = requests.get(url, headers={"Accept": "application/json"}, timeout=60) + page.raise_for_status() + data = page.json() + rows += data["results"] + url = (data.get("next") or "").replace("http://", "https://") + path.write_text(json.dumps(rows), encoding="utf-8") + return rows + + def get_or_create_bw_process(dpp_product): raise NotImplementedError() return +def get_biosphere_db(): + """Get the biosphere database of the currently active project.""" + databases = list(bwd.databases) + for name in [BIOSPHERE, "ef-3.1-biosphere"]: + if name in databases: + return bwd.Database(name) + for name in databases: + if "biosphere" in name: + return bwd.Database(name) + raise FileNotFoundError(f"No biosphere database found in {bwd.projects.current}.") + def biosphere_to_dpp_flows(): """Convert all biosphere flows to Emission objects""" from .models import Emission @@ -106,8 +259,8 @@ def biosphere_to_dpp_flows(): 'square meter-year': 'm2-yr', 'standard cubic meter': 'Nm3', } - setup_project("L4M-DPP") - biosphere_db = bwd.Database(DEFAULT_REMOTE_PROJECT) + setup_project() + biosphere_db = get_biosphere_db() bio_flows = set((i['name'], i['unit']) for i in list(biosphere_db)) if len(bio_flows) <= len(Emission.objects.all()): return for name, unit in bio_flows: @@ -166,6 +319,9 @@ def make_transport_exchange(transports, product, amount): transports: QuerySet of Transport table product: ProductModel, input to some process amount: int, amount of product transported + + Returns: + Brightway exchange, describing transport input """ # Import here to avoid circular imports from .models import UNIT_CHOICES, CONVERSIONS @@ -178,6 +334,24 @@ def make_transport_exchange(transports, product, amount): "delivery van": '558949a7ddfcccba760eab39bda68e88', # light commercial vehicle, EU "NA": 'dd736ab7d965646d04b091ae5fa68d97', #FIXME defaults to truck for now } + bafu_codes = { + "train": 'aacc6120-0e68-3835-8303-5ae37d62c030', # freight train, average, EU + "ocean ship": 'dc299788-d8c5-3adc-b025-d607edd4d973', # sea freight container ship, GLO + "truck": '99f8a4b0-99b5-3130-924c-5de4c10b62ef', # lorry, fleet average, RER + "inland ship": 'missing', # inland waterways, RER + "airplane": '1313d894-5c0a-38cd-b06c-fcce93e52cf9', # aircraft, medium-haul, GLO + "delivery van": 'c21996a4-ec68-361e-8fda-dc3ad1081e7d', #FIXME Operation, van, CH + "NA": '99f8a4b0-99b5-3130-924c-5de4c10b62ef', #FIXME defaults to truck for now + } + bonsai_codes = { + "train": "M_TLAND_ALL", + "ocean ship": "M_TWATR_ALL", + "truck": "M_TLAND_ALL", + "inland ship": "M_TWATR_ALL", + "airplane": "C_TAIR", + "delivery van": "M_TLAND_ALL", + "NA": "M_TLAND_ALL", + } transport = transports.filter(product=product).first() if transport is None: raise LookupError(f"No transport specified for {product}") @@ -192,12 +366,52 @@ def make_transport_exchange(transports, product, amount): else: mass = 1 - return { # Calculate transport distance (t-km) - "input": (ECOINVENT, ecoinvent_codes[transport.mode]), - "amount": mass / 1000 * amount * transport.distance, - "type": "technosphere", - "unit": 'ton kilometer', - } + # Calculate transport distance (t-km) or price (MEuro) + if ECOINVENT in bwd.databases: + exchange = { + "input": (ECOINVENT, ecoinvent_codes[transport.mode]), + "amount": mass / 1000 * amount * transport.distance, + "type": "technosphere", + "unit": 'ton kilometer', + } + elif BAFU_DB in bwd.databases: + exchange = { + "input": (BAFU_DB, ecoinvent_codes[transport.mode]), + "amount": mass / 1000 * amount * transport.distance, + "type": "technosphere", + "unit": 'ton kilometer', + } + elif BONSAI_DB in bwd.databases: + transp_code = bonsai_codes[transport.mode] + price = country_data.loc["?", transp_code + "_price"] #FIXME: country + exchange = { + "input": (BONSAI_DB, transp_code), + "amount": mass * amount * transport.distance * price / 1e9, + # units: kg/pcs * pcs * km * EUR/t-km -> MEuro + "type": "technosphere", + "unit": 'MEUR', + } + return exchange + +def list_background_processes() -> dict: + """ + Create a dictionary of available background processes + Structured as: {database_name: {activity_name: activity_code}} + """ + setup_project() + background = {} + for db_name in bwd.databases: + if db_name[:4] != "dpp_": + background[db_name] = { + act["name"]: act["code"] + for act in bwd.Database(db_name) + } + return background + + +# --------------------------------------------------------------------------- +# Create DPP LCA and do LCA calculations +# --------------------------------------------------------------------------- def convert_dpp_to_brightway(processes: list, db_name: str): """ @@ -209,7 +423,7 @@ def convert_dpp_to_brightway(processes: list, db_name: str): # Import here to avoid circular imports from .models import ProductExchange, EnvExchange, BackgroundProcess - biosphere = bwd.Database(DEFAULT_REMOTE_PROJECT) + biosphere = get_biosphere_db() bw_activities = {} for dpp_process in processes: # Skip already known background processes @@ -339,7 +553,7 @@ def create_supply_chain_lca(product): # Import here to avoid circular imports from .models import SustainabilityEvaluation, SustainabilityScore, ImpactIndicator - setup_project("L4M-DPP") + setup_project() lcia_family = 'EF v3.1' method_set = ensure_methods(lcia_family) evaluation, created = SustainabilityEvaluation.objects.get_or_create( @@ -411,18 +625,3 @@ def create_supply_chain_lca(product): ) return evaluation - -def list_background_processes() -> dict: - """ - Create a dictionary of available background processes - Structured as: {database_name: {activity_name: activity_code}} - """ - setup_project("L4M-DPP") - background = {} - for db_name in bwd.databases: - if db_name[:4] != "dpp_": - background[db_name] = { - act["name"]: act["code"] - for act in bwd.Database(db_name) - } - return background diff --git a/mysite/dpp/models.py b/mysite/dpp/models.py index 7838023..026a0d9 100644 --- a/mysite/dpp/models.py +++ b/mysite/dpp/models.py @@ -466,16 +466,18 @@ def disassemble(self): Returns a list of created ProductItems. """ created_items = [] + entries = self.product_batch.model.composed_of.select_related('component') - for i, component in enumerate(self.components.all()): - component_serial = f"{self.serial_number}-C{i}" - for j in range(component.amount): + for i, entry in enumerate(entries): + part_batch = entry.component.productbatch + for j in range(entry.amount): # Generate unique serial number for each component - if component.amount > 1: + component_serial = f"{self.serial_number}-C{i}" + if entry.amount > 1: component_serial += f"-{j}" new_item = ProductItem.objects.create( - product_batch=component, + product_batch=part_batch, serial_number=component_serial, production_date=self.production_date, ) @@ -664,8 +666,8 @@ def aggregate_production(self): pl_map = {} #FIXME: unused ManufacturingProcess to Process map for p in other_source: pl = p.production_line - mp = pl.create_public_tables() - suppliers += [mp] + mp = pl.aggregate_production() + suppliers = list(suppliers) + [mp] pl_map[mp.pk] = p.pk exchanges = ( diff --git a/mysite/dpp/tests/test_lca.py b/mysite/dpp/tests/test_lca.py index 62d59d9..0eb948c 100644 --- a/mysite/dpp/tests/test_lca.py +++ b/mysite/dpp/tests/test_lca.py @@ -1,20 +1,23 @@ """ Tests for lca.py functions: - setup_project, ensure_methods, select_supply_chain, + setup_project, ensure_methods, make_transport_exchange, select_supply_chain, convert_dpp_to_brightway, lca_calculations, create_supply_chain_lca All Brightway and Django ORM calls are mocked so the suite runs without -a live database or ecoinvent licence. - -Run with: - pytest test_lca.py -v +a live database or Ecoinvent licence. """ -from django.test import TestCase +import uuid +import bw2data as bwd +import pandas as pd + +from django.test import SimpleTestCase, TestCase import datetime -from unittest.mock import MagicMock, patch -from ..models import IndicatorSet +from types import SimpleNamespace +from unittest.mock import Mock, MagicMock, patch +from ..models import CONVERSIONS, IndicatorSet, ProductModel, Transport from dpp.lca import ( - EXCLUDED_METHODS, setup_project, ensure_methods, select_supply_chain, + EXCLUDED_METHODS, BIOSPHERE, ECOINVENT, BAFU_DB, BONSAI_DB, + setup_project, ensure_methods, make_transport_exchange, select_supply_chain, convert_dpp_to_brightway, lca_calculations, create_supply_chain_lca, ) @@ -33,7 +36,6 @@ def make_flow(pk=1, name="product", unit="kg", produced_by=None): flow.manufacturing_info = produced_by return flow - def make_process(pk=1, name="proc", unit="kg", facility=None, description="desc", amount=1.0, database="mydb"): proc = MagicMock() @@ -52,31 +54,98 @@ def make_process(pk=1, name="proc", unit="kg", facility=None, # Test classes for key LCA functions # --------------------------------------------------------------------------- -class TestSetupProject(TestCase): +class TestSetupProject(SimpleTestCase): + def setUp(self): + self.project = "LCA-test-project" + self.addCleanup(self._delete_project) + + # autospec=True enforces the real signatures, so a wrong call raises TypeError + self.install = self._patch("dpp.lca.bwi.remote.install_project") + self.import_db = self._patch("dpp.lca.import_db") + self.import_bonsai = self._patch("dpp.lca.import_bonsai") + self.status = self._patch("dpp.lca.project_status") + self.set_status(biosphere=True, bonsai=True) + + def _patch(self, target): + patcher = patch(target, autospec=True) + mocked = patcher.start() + self.addCleanup(patcher.stop) + return mocked + + def _delete_project(self): + if self.project in bwd.projects: + bwd.projects.set_current("default") + bwd.projects.delete_project(self.project, delete_dir=True) + + def set_status(self, **status): + for key, name in [ + ("biosphere", BIOSPHERE), ("bafu", BAFU_DB), ("bonsai", BONSAI_DB) + ]: + if key in status: + status[name] = status[key] + del status[key] + self.status.return_value = status + + def test_missing_biosphere_installs_it(self): + self.set_status(biosphere=False, bonsai=True) + setup_project(self.project, background="BONSAI") + self.install.assert_called_once_with( + project_key=BIOSPHERE, project_name=self.project, overwrite_existing=True + ) + self.import_bonsai.assert_not_called() + + def test_missing_bonsai_imports_it(self): + self.set_status(biosphere=True, bonsai=False) + setup_project(self.project, background="BONSAI") + self.import_bonsai.assert_called_once_with() + self.install.assert_not_called() + + def test_missing_bafu_imports_it(self): + self.set_status(biosphere=True, bonsai=False) + setup_project(self.project, background="BAFU") + self.import_db.assert_called_once_with(project=self.project, overwrite=False) + self.install.assert_not_called() + + def test_empty_status_imports_both(self): + # e.g. the project does not exist yet: no keys at all + self.set_status() + setup_project(self.project, background="BAFU") + self.import_db.assert_called_once_with(project=self.project, overwrite=False) + setup_project(self.project, background="BONSAI") + self.install.assert_called_once_with( + project_key=BIOSPHERE, project_name=self.project, overwrite_existing=True + ) + self.import_bonsai.assert_called_once_with() + self.assertEqual(bwd.projects.current, self.project) + + def test_biosphere_is_installed_before_background_import(self): + self.set_status(biosphere=False, bonsai=False) + manager = Mock() + manager.attach_mock(self.install, "install") + manager.attach_mock(self.import_bonsai, "import_bonsai") + setup_project(self.project, background="BONSAI") + self.assertEqual( + [c[0] for c in manager.mock_calls], ["install", "import_bonsai"] + ) - @patch("dpp.lca.bwi") - @patch("dpp.lca.bwd") - def test_create_new_project(self, mock_bwd, mock_bwi): - """When the project doesn't exist it should be created and configured.""" - mock_bwd.projects.__contains__ = MagicMock(return_value=False) - mock_bwd.projects.current = "L4M-test" - setup_project("L4M-test") - - mock_bwd.projects.set_current.assert_called_once_with("L4M-test") - mock_bwi.remote.install_project.assert_called_once_with( - "ecoinvent-3.12-biosphere", "L4M-test" + def test_incomplete_bonsai_is_removed_before_import(self): + self.set_status(biosphere=True, bonsai=False) + bwd.projects.set_current(self.project) + bwd.Database(BONSAI_DB).write({}) # registered, but not processed + present_at_import = [] + self.import_bonsai.side_effect = lambda: present_at_import.append( + BONSAI_DB in bwd.databases ) - @patch("dpp.lca.bwi") - @patch("dpp.lca.bwd") - def test_reuse_existing_project(self, mock_bwd, mock_bwi): - """When the project already exists only set_current should be called.""" - mock_bwd.projects.__contains__ = MagicMock(return_value=True) - mock_bwd.projects.current = "L4M-test" - setup_project("L4M-test") + setup_project(self.project, background="BONSAI") + self.assertEqual(present_at_import, [False]) - mock_bwd.projects.set_current.assert_called_once_with("L4M-test") - mock_bwi.remote.install_project.assert_not_called() + def test_complete_project_installs_nothing(self): + self.set_status(biosphere=True, bafu=True) + setup_project(self.project) + self.install.assert_not_called() + self.import_db.assert_not_called() + self.assertEqual(bwd.projects.current, self.project) class TestEnsureMethods(TestCase): @@ -209,8 +278,18 @@ def test_max_depth_limits_traversal(self): class TestConvertDppToBrightway(TestCase): - @patch("dpp.lca.bwd") - def test_output_contains_activity_key(self, mock_bwd): + def setUp(self): + bwd_patcher = patch("dpp.lca.bwd") + biosphere_patcher = patch("dpp.lca.get_biosphere_db") + + self.mock_bwd = bwd_patcher.start() + self.mock_get_biosphere_db = biosphere_patcher.start() + self.addCleanup(bwd_patcher.stop) + self.addCleanup(biosphere_patcher.stop) + + self.mock_bwd.databases = {} # no pre-existing databases + + def test_output_contains_activity_key(self): """Each process must produce a key of the form (db_name, pk).""" proc = make_process(pk=42, name="my proc", unit="kg") @@ -225,8 +304,7 @@ def test_output_contains_activity_key(self, mock_bwd): assert ("testdb", 42) in result - @patch("dpp.lca.bwd") - def test_production_exchange_is_present(self, mock_bwd): + def test_production_exchange_is_present(self): """The activity must always have a production exchange.""" proc = make_process(pk=1, unit="kg", amount=2.0) @@ -243,8 +321,7 @@ def test_production_exchange_is_present(self, mock_bwd): assert len(prod_excs) == 1 assert prod_excs[0]["amount"] == 2.0 - @patch("dpp.lca.bwd") - def test_stage_raw_material_for_mass_unit(self, mock_bwd): + def test_stage_raw_material_for_mass_unit(self): """Processes with a mass-unit functional flow should be 'Raw material acquisition'.""" proc = make_process(pk=1, unit="kg") @@ -256,8 +333,7 @@ def test_stage_raw_material_for_mass_unit(self, mock_bwd): assert result[("db", 1)]["stage"] == "Raw material acquisition" - @patch("dpp.lca.bwd") - def test_stage_manufacturing_for_non_resource_unit(self, mock_bwd): + def test_stage_manufacturing_for_non_resource_unit(self): """Processes with unit 'pcs' (not a resource unit) should be 'Manufacturing'.""" proc = make_process(pk=1, unit="pcs") @@ -269,8 +345,7 @@ def test_stage_manufacturing_for_non_resource_unit(self, mock_bwd): assert result[("db", 1)]["stage"] == "Manufacturing" - @patch("dpp.lca.bwd") - def test_cutoff_excludes_external_products(self, mock_bwd): + def test_cutoff_excludes_external_products(self): """ProductExchanges whose upstream process is not in `processes` are cut off.""" proc = make_process(pk=1, unit="pcs") external_proc = make_process(pk=99) @@ -290,6 +365,76 @@ def test_cutoff_excludes_external_products(self, mock_bwd): tech_excs = [e for e in result[("db", 1)]["exchanges"] if e["type"] == "technosphere"] assert len(tech_excs) == 0 +class MockTransports: + """ + Minimal stand-in for the Transport QuerySet. + supports .filter(product=...).first(). + """ + def __init__(self, *items): + self._items = list(items) + + def filter(self, product): + return MockTransports(*[t for t in self._items if t.product == product]) + + def first(self): + return self._items[0] if self._items else None + +class TestMakeTransportExchange(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Brightway state lives outside the Django DB, so TestCase's + # transaction rollback doesn't cover it: manage it explicitly. + cls.bw_project = f"test-{uuid.uuid4().hex}" + bwd.projects.set_current(cls.bw_project) + cls.addClassCleanup(bwd.projects.delete_project, cls.bw_project, delete_dir=True) + bwd.Database(BONSAI_DB).write({ + (BONSAI_DB, "M_TLAND_ALL"): { + "name": "Land transport", + "unit": "MEUR", + "location": "GLO", + "exchanges": [], + }, + }) + + def setUp(self): + bwd.projects.set_current(self.bw_project) + self.assertNotIn(ECOINVENT, bwd.databases) # forces the BONSAI branch + + self.mock_prices = pd.DataFrame({"M_TLAND_ALL_price": [0.1]}, index=["?"]) + + self.distance_km = 500 + self.product = SimpleNamespace(name="widget", unit="kg") # no `properties` -> uses unit + self.transports = MockTransports( + SimpleNamespace(product=self.product, mode="truck", distance=self.distance_km) + ) + + def test_exchange_values(self): + amount = 200 # units of product + with patch("dpp.lca.country_data", self.mock_prices): + exchange = make_transport_exchange( + self.transports, self.product, amount + ) + + mass_kg = CONVERSIONS["kg"] + price = self.mock_prices.loc["?", "M_TLAND_ALL_price"] # EUR/tkm + expected = mass_kg / 1000 * amount * self.distance_km * price / 1e6 # tkm * EUR/tkm -> MEUR + + self.assertEqual(exchange["input"], (BONSAI_DB, "M_TLAND_ALL")) + self.assertEqual(exchange["type"], "technosphere") + self.assertEqual(exchange["unit"], "MEUR") + self.assertAlmostEqual(exchange["amount"], expected) + + def test_exchange_points_to_existing_activity(self): + with patch("dpp.lca.country_data", self.mock_prices): + exchange = make_transport_exchange(self.transports, self.product, 1) + self.assertEqual(bwd.get_activity(exchange["input"])["unit"], "MEUR") + + def test_missing_transport_raises(self): + other = ProductModel.objects.create(name="gadget", unit="kg") + with patch("dpp.lca.country_data", self.mock_prices): + with self.assertRaises(LookupError): + make_transport_exchange(self.transports, other, 1) class TestLcaCalculations(TestCase): diff --git a/mysite/dpp/tests/test_models.py b/mysite/dpp/tests/test_models.py index 9a34afb..b318628 100644 --- a/mysite/dpp/tests/test_models.py +++ b/mysite/dpp/tests/test_models.py @@ -1,3 +1,5 @@ +from collections import Counter +import re import uuid from unittest.mock import patch @@ -62,6 +64,12 @@ def add_properties(product, weight=1.0, unit="kg", **kwargs): def make_batch(model, number=1, gtin="0000000000001"): return ProductBatch.objects.create(batch_number=number, model=model, GTIN=gtin) +def make_emission(process, substance, amount, compartment="air"): + return EnvExchange.objects.create( + process=process, substance=substance, compartment=compartment, + direction="out", amount=amount, + ) + class SupplyChainTestCase(TestCase): """ @@ -341,8 +349,55 @@ def test_update_circularity_rejects_unknown_code(self): self.item.refresh_from_db() self.assertEqual(self.item.circularity, "new") - # NOTE: ProductItem.disassemble() is not tested: it uses `self.components` - # and `GTIN_code`, neither of which exists on the model. + # -- disassemble -------------------------------------------------------- + def _add_parts(self, *amounts): + """ + Give the item's product model one part (a ProductBatch) per amount. + Returns the part batches, in order. + """ + parent = self.item.product_batch.model + batches = [] + for n, amount in enumerate(amounts): + batch = make_batch(make_product(f"Part {n}"), gtin=f"10000000000{n:02d}") + Component.objects.create(product=parent, component=batch, amount=amount) + batches.append(batch) + return batches + + def test_disassemble_creates_one_item_per_component_unit(self): + screws, motor = self._add_parts(3, 1) + + created = self.item.disassemble() + + self.assertEqual(len(created), 4) + self.assertEqual(ProductItem.objects.count(), 5) # the original + 4 parts + per_batch = Counter(item.product_batch_id for item in created) + self.assertEqual(per_batch, {screws.pk: 3, motor.pk: 1}) + + def test_disassemble_without_components_creates_nothing(self): + self.assertEqual(self.item.disassemble(), []) + self.assertEqual(ProductItem.objects.count(), 1) + + def test_disassemble_parts_inherit_production_date(self): + self._add_parts(2) + self.item.production_date = datetime.date(2024, 1, 15) + self.item.save() + + created = self.item.disassemble() + + self.assertTrue(created) + for part in created: + self.assertEqual(part.production_date, datetime.date(2024, 1, 15)) + + def test_disassemble_serial_numbers_are_unique_and_derived_from_parent(self): + self._add_parts(3, 1) + created_items = self.item.disassemble() + + serials = [item.serial_number for item in created_items] + self.assertEqual(len(set(serials)), len(serials)) + multi = [s for s in serials if re.fullmatch(r"SN-1-C\d+-\d+", s)] + single = [s for s in serials if re.fullmatch(r"SN-1-C\d+", s)] + self.assertEqual(len(multi), 3) # amount > 1: numbered per unit + self.assertEqual(len(single), 1) # amount == 1: no unit suffix class ComponentTests(TestCase): @@ -384,36 +439,6 @@ def test_is_hazardous_for_hazardous_material(self): # Activities, exchanges # -------------------------------------------------------------------------- -class ActivityBiosphereTests(TestCase): - def test_aggregate_biosphere_scales_and_sums_exchanges(self): - p1 = Activity.objects.create(name="p1") - p2 = Activity.objects.create(name="p2") - target = Activity.objects.create(name="aggregated") - co2 = Emission.objects.create(name="CO2") - ch4 = Emission.objects.create(name="CH4") - - def env(process, substance, amount, compartment="air"): - return EnvExchange.objects.create( - process=process, substance=substance, compartment=compartment, - direction="out", amount=amount, - ) - env(p1, co2, 1) - env(p1, ch4, 4) - env(p2, co2, 2) - env(target, ch4, 99, compartment="soil") # stale, must be removed - - target.aggregate_biosphere(pd.Series({p1.id: 2.0, p2.id: 3.0})) - - result = { - (e.substance.name, e.compartment): e.amount - for e in EnvExchange.objects.filter(process=target) - } - self.assertEqual(set(result), {("CO2", "air"), ("CH4", "air")}) - self.assertAlmostEqual(result[("CO2", "air")], 2 * 1 + 3 * 2) - self.assertAlmostEqual(result[("CH4", "air")], 2 * 4) - # Source exchanges are untouched - self.assertEqual(EnvExchange.objects.get(process=p1, substance=co2).amount, 1) - class ManufacturingProcessTests(TestCase): def test_save_requires_facility(self): @@ -451,15 +476,20 @@ def test_functional_flow_is_required(self): proc.save() -class ProductExchangeTests(TestCase): +class ExchangeTests(TestCase): def setUp(self): self.process = Activity.objects.create(name="act") + self.substance = Emission.objects.create(name="CO2") def test_type_defaults_from_direction(self): ex_in = ProductExchange.objects.create( - process=self.process, product=make_product("a"), amount=1, direction="in") + process=self.process, product=make_product("a"), + amount=1, direction="in", + ) ex_out = ProductExchange.objects.create( - process=self.process, product=make_product("b"), amount=1, direction="out") + process=self.process, product=make_product("b"), + amount=1, direction="out", + ) self.assertEqual(ex_in.type, "prod") self.assertEqual(ex_out.type, "waste") @@ -469,12 +499,6 @@ def test_output_must_be_waste(self): process=self.process, product=make_product(), amount=1, direction="out", type="cons") - -class EnvExchangeTests(TestCase): - def setUp(self): - self.process = Activity.objects.create(name="act") - self.substance = Emission.objects.create(name="CO2") - def test_functional_flow_direction_is_rejected(self): with self.assertRaises(ValidationError): EnvExchange.objects.create( @@ -542,10 +566,34 @@ def setUp(self): name="Raw production", amount=1, facility=self.facility, functional_flow=self.raw) self.co2 = Emission.objects.create(name="CO2") - for proc, amount in [(self.p1, 1), (self.p2, 2)]: - EnvExchange.objects.create( - process=proc, substance=self.co2, compartment="air", - direction="out", amount=amount) + make_emission(self.p1, self.co2, 1) + make_emission(self.p2, self.co2, 2) + + def test_aggregate_biosphere_exchanges(self): + """Test the Activity function `aggregate_biosphere`, + that should scale and aggregates environmental exchanges. + """ + target = Activity.objects.create(name="aggregated") + ch4 = Emission.objects.create(name="CH4") + co2 = self.co2 + + # Add 2 environmental exchanges, additional to those in setUp + make_emission(self.p1, ch4, 4) + make_emission(target, ch4, 99, compartment="soil") # stale, must be removed + + target.aggregate_biosphere(pd.Series({self.p1.id: 2.0, self.p2.id: 3.0})) + + result = { + (e.substance.name, e.compartment): e.amount + for e in EnvExchange.objects.filter(process=target) + } + self.assertEqual(set(result), {(co2.name, "air"), (ch4.name, "air")}) + self.assertAlmostEqual(result[(co2.name, "air")], 2 * 1 + 3 * 2) + self.assertAlmostEqual(result[(ch4.name, "air")], 2 * 4) + # Source exchanges are untouched + self.assertEqual( + EnvExchange.objects.get(process=self.p1, substance=co2).amount, 1 + ) def test_aggregate_production_creates_manufacturing_process(self): agg = self.line.aggregate_production() diff --git a/mysite/dss/static/kam_input_funcs.js b/mysite/dss/static/kam_input_funcs.js index 42b3905..af63753 100644 --- a/mysite/dss/static/kam_input_funcs.js +++ b/mysite/dss/static/kam_input_funcs.js @@ -244,7 +244,7 @@ function buildPayload() { { name: 'Operator specialisation level', value: gl('specialisation'), target: 'min' }, { name: 'Process monitoring', value: gi('monitoring'), target: 'min' }, { name: 'Production lead time', value: gi('leadTime'), target: 'min' }, - { name: 'Machine saturation', value: gf('saturation'), target: 'max' }, + { name: 'Machine saturation', value: gf('saturation'), target: 'min' }, ], }; }); diff --git a/mysite/dss/tests.py b/mysite/dss/tests.py index c5cb460..f64fd3e 100644 --- a/mysite/dss/tests.py +++ b/mysite/dss/tests.py @@ -208,7 +208,6 @@ def test_calculation_result(self, pd_experiment, kam_experiment): result = calculate_experiment_kpis(kam_experiment, "kam") assert result["Process energy use"] > pd_result["Process energy use"] - assert result["Total production costs"] > result["Operating costs"] def test_scrap_rate_of_one_raises_zero_division(self, kam_experiment): @@ -235,7 +234,6 @@ def test_calculate_kpis_propagates_unrecognised_unit_error(pd_experiment): @pytest.mark.django_db class TestCalculateKpis: - def test_creates_session_and_matrix(self, pd_experiment): exp2 = dict(pd_experiment, experimentId=2) valid_data = [pd_experiment, exp2] diff --git a/mysite/init_data/country_data.csv b/mysite/init_data/country_data.csv index 1f9a6ac..827375a 100644 --- a/mysite/init_data/country_data.csv +++ b/mysite/init_data/country_data.csv @@ -1,3 +1,43 @@ -country,wages,electricity_price,electricity_CO2eq -DE,60,0.25,21 -?,70,0.214,23 +country,wages,electricity_price,electricity_CO2eq +AL,0,0.1644,0 +AT,51.3,0.24085,116.605 +BA,0,0.1303,572.891 +BE,0,0.2303,151.262 +BG,10.2,0.16105,275.347 +CY,14.6,0.2802,505.196 +CZ,20.2,0.222,401.366 +DE,49.5,0.27335,334.117 +DK,55,0.2725,99.887 +EE,19.5,0.17495,31.856 +EL,18.2,0.19435,324.762 +ES,28.2,0.16385,153.459 +?,35,0.22445,210.596 +FI,43.6,0.096,57.242 +FR,47.1,0.1886,41.224 +GE,0,0.09955,0 +HR,19.2,0.18565,158.091 +HU,15.6,0.26665,16.262 +IE,42.4,0.2872,263.289 +IS,61.2,0,0 +IT,32.8,0.2625,284.431 +LI,0,0.19385,0 +LT,16.4,0.1926,138.142 +LU,48.4,0.1891,118.347 +LV,14.9,0.1655,134.896 +MD,0,0.19595,0 +ME,0,0.105,274.897 +MK,0,0.1479,426.061 +MT,20,0.1423,483.393 +NL,0,0.2486,253.664 +NO,53.8,0.1029,2.805 +PL,17.1,0.24175,590.814 +PT,16.5,0.16145,1.278 +RO,12,0.2161,259.166 +RS,12,0.17505,699.065 +SE,47.2,0.1209,0.354 +SI,29.8,0.18655,188.482 +SK,19.3,0.2414,95.279 +TR,8.2,0.1072,475.776 +UA,0,0,0 +UK,0,0,218.207 +XK,0,0.1831,905.525 diff --git a/mysite/init_data/material_data.csv b/mysite/init_data/material_data.csv index c8057b7..762d416 100644 --- a/mysite/init_data/material_data.csv +++ b/mysite/init_data/material_data.csv @@ -1,6 +1,16 @@ material,price,CO2eq,unit -Argon,13,9.47,m3 -Nitrogen,7.05,7.11,m3 -Aluminium (filler wire),100,88,kg -Steel,56,42,kg -Stainless steel,56,42,kg +"Aluminium alloy, cast",2.74,6.165,kg +"Aluminium, virgin",2.74,16.546,kg +"Aluminium alloy, wrought",2.74,12.843,kg +Aluminium filler wire,26.17,12.843,kg +Aluminium sheet,3.10,13.463,kg +Chromium steel pipe,3.50,5.424,kg +AISI 316 steel plate,3.50,5.424,kg +"Steel, low-alloyed",0.41,2.024,kg +"Steel, low-alloyed, hot rolled",1.17,2.279,kg +Argon,10.47,1.279,m3 +Nitrogen,6.69,0.201,m3 +Helium,97.16,7.847,m3 +Chromium steel sheets,0.576345982,5.815,kg +Steel wire,20.76,2.633,kg +Steel sheets,0.55900772,2.584,kg diff --git a/mysite/scripts/data_imports.py b/mysite/scripts/data_imports.py index f438cc6..98893b5 100644 --- a/mysite/scripts/data_imports.py +++ b/mysite/scripts/data_imports.py @@ -5,6 +5,7 @@ from django.conf import settings # from django.contrib.auth.models import User from dpp.models import CircularityIndicator, Instruction, ImpactIndicator, ImpactCategory +from dpp.lca import biosphere_to_dpp_flows def csv_to_django(file_path: Path | str, Model, relations={}): """ @@ -36,3 +37,4 @@ def run(): socioecon, _ = ImpactCategory.objects.get_or_create(name="Socio-economic impact") socioecon_csv = settings.DATA_DIR / "socioecon_indicators.csv" csv_to_django(socioecon_csv, ImpactIndicator, relations={"impact_category": socioecon.pk}) + biosphere_to_dpp_flows() diff --git a/requirements.txt b/requirements.txt index 9e81db1..e405d5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ environs pandas psycopg2 brightway25 +sentier_brightway @ git+https://github.com/sentier-dev/sentier-brightway