diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 981a41d..e12b296 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -33,7 +33,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - py-version: ["3.9", "3.10", "3.11", "3.12"] + py-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index b6e4761..0a10c95 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,4 @@ dmypy.json # Pyre type checker .pyre/ +pyrightconfig.json diff --git a/CHANGES.md b/CHANGES.md index 399cb9c..59f8d79 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,16 +1,36 @@ # flowmapper Changelog -All notable changes to this project will be documented in this file. +## [0.4.2] - 2025-07-28 -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +* Remove conflicting synonyms when parsing ecospold2 +* Prioritize ionic-form synonyms over more generic synonyms -## [Unreleased] +## [0.4.1] - 2025-07-18 -## [0.1.0] - 2022-03-22 +* Updated some manual SimaPro - ecoinvent 3.10 matches -### Added +## [0.4] - 2025-03-03 -### Changed +* Add ability to build manual matchings as input transformations -### Removed +### [0.3.1] - 2025-02-21 + +* Minor fixes to input data +* Minor fixes to matching functions + +## [0.3] - 2024-12-02 + +* Add more command line options +* Small bugfixes + +## [0.2] - 2024-09-04 + +* Rewrote most of code base + +### [0.1.1] - 2024-05-07 + +* Packaging fix + +## [0.1.0] - 2024-05-07 + +* Initial release diff --git a/Generate ecoinvent transitive mapping.ipynb b/Generate ecoinvent transitive mapping.ipynb new file mode 100644 index 0000000..bb14887 --- /dev/null +++ b/Generate ecoinvent transitive mapping.ipynb @@ -0,0 +1,619 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "d8e9c0b3-dfa9-46cc-b973-5fe953f38521", + "metadata": {}, + "outputs": [], + "source": [ + "import randonneur_data as rd\n", + "import randonneur as rn\n", + "from flowmapper.extraction.ecospold2 import remove_conflicting_synonyms, reformat\n", + "from pathlib import Path\n", + "import xmltodict\n", + "import structlog\n", + "import logging\n", + "from pathlib import Path\n", + "from tqdm import tqdm\n", + "from copy import deepcopy\n", + "from collections import defaultdict" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "303ee863-eabf-42ff-bef7-cb09e654bc7e", + "metadata": {}, + "outputs": [], + "source": [ + "logging.config.dictConfig({\n", + " \"version\": 1,\n", + " \"disable_existing_loggers\": False,\n", + " \"handlers\": {\n", + " \"file\": {\n", + " \"level\": \"DEBUG\",\n", + " \"class\": \"logging.handlers.WatchedFileHandler\",\n", + " \"filename\": \"test.log\",\n", + " },\n", + " },\n", + " \"loggers\": {\n", + " \"\": {\n", + " \"handlers\": [\"file\"],\n", + " \"level\": \"DEBUG\",\n", + " \"propagate\": True,\n", + " },\n", + " }\n", + "})\n", + "structlog.configure(\n", + " processors=[\n", + " structlog.stdlib.filter_by_level,\n", + " structlog.stdlib.add_logger_name,\n", + " structlog.stdlib.add_log_level,\n", + " structlog.stdlib.PositionalArgumentsFormatter(),\n", + " structlog.processors.TimeStamper(fmt=\"iso\"),\n", + " structlog.processors.StackInfoRenderer(),\n", + " structlog.processors.format_exc_info,\n", + " structlog.processors.UnicodeDecoder(),\n", + " structlog.processors.JSONRenderer(),\n", + " structlog.stdlib.ProcessorFormatter.wrap_for_formatter,\n", + " ],\n", + " logger_factory=structlog.stdlib.LoggerFactory(),\n", + " wrapper_class=structlog.stdlib.BoundLogger,\n", + " cache_logger_on_first_use=True,\n", + ")\n", + "logger = structlog.get_logger(\"ecoinvent-migrate\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e80afdee-7e95-4cb4-8c9b-eb85aed4312e", + "metadata": {}, + "outputs": [], + "source": [ + "registry = rd.Registry()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6890a48b-9a26-4365-8e3d-1d9bcfaa69f5", + "metadata": {}, + "outputs": [], + "source": [ + "def get_elem_flow_data(version: str) -> list[dict]:\n", + " path = Path(f\"/Users/cmutel/Library/Application Support/EcoinventInterface/cache/ecoinvent {version}_cutoff_ecoSpold02/MasterData/ElementaryExchanges.xml\")\n", + "\n", + " if not path.is_file():\n", + " path = Path(f\"/Users/cmutel/Library/Application Support/EcoinventInterface/cache/ecoinvent {version}_cut-off_ecoSpold02/MasterData/ElementaryExchanges.xml\")\n", + " \n", + " with open(path) as fs:\n", + " ei_xml = xmltodict.parse(fs.read(), strip_whitespace=False)[\n", + " \"validElementaryExchanges\"\n", + " ][\"elementaryExchange\"]\n", + "\n", + " data = remove_conflicting_synonyms([reformat(obj) for obj in ei_xml])\n", + "\n", + " for obj in data:\n", + " if \"formula\" in obj:\n", + " del obj[\"formula\"]\n", + " if \"cas_number\" in obj and not obj[\"cas_number\"]:\n", + " del obj[\"cas_number\"]\n", + "\n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c6617ef6-54d4-4c9d-bb98-c4a7dfed81a3", + "metadata": {}, + "outputs": [], + "source": [ + "def get_elem_flow_dict(version: str) -> dict:\n", + " return {row['identifier']: row for row in get_elem_flow_data(version)}" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "35fd877d-ebe2-45be-9381-d49fdcfa067a", + "metadata": {}, + "outputs": [], + "source": [ + "def add_comment(comment: str | None, addition: str, deletions: list[str] = [\"replaced\"]) -> str:\n", + " if comment is None:\n", + " comment = \"\"\n", + " \n", + " for deletion in deletions:\n", + " if comment == deletion:\n", + " comment = \"\"\n", + "\n", + " if comment and not comment.endswith(\".\"):\n", + " comment += \".\"\n", + "\n", + " if comment:\n", + " return comment + \" \" + addition\n", + " else:\n", + " return addition" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "49971a58-c14f-4480-86c5-698416308380", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_changes(\n", + " source_version: str, \n", + " target_version: str\n", + ") -> list[dict]:\n", + " \"\"\"Use the `uuid` in the change list to add in other attributes not given in change list.\"\"\"\n", + " source_flow_dict = get_elem_flow_dict(source_version)\n", + " target_flow_dict = get_elem_flow_dict(target_version)\n", + "\n", + " changes = []\n", + "\n", + " for s_key, s_data in source_flow_dict.items():\n", + " if s_key not in target_flow_dict:\n", + " logger.debug(\"Elementary flow deleted: %s\", s_data)\n", + " continue\n", + "\n", + " t_data = target_flow_dict[s_key]\n", + " \n", + " if t_data == s_data:\n", + " continue\n", + "\n", + " attributes = \", \".join([\n", + " key \n", + " for key, value in t_data.items() \n", + " if key in s_data \n", + " and s_data[key] != value\n", + " ])\n", + " change = {\n", + " \"source\": s_data,\n", + " \"target\": t_data,\n", + " \"comment\": f\"Changed {attributes} from {source_version} to {target_version}.\",\n", + " \"source_version\": f\"ecoinvent-{source_version}-biosphere\",\n", + " \"target_version\": f\"ecoinvent-{target_version}-biosphere\"\n", + " }\n", + " changes.append(change)\n", + "\n", + " return changes " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fe2a20c7-88ea-4019-a657-67110684532c", + "metadata": {}, + "outputs": [], + "source": [ + "def supplement_changes(\n", + " changes: list[dict], \n", + " source_version: str, \n", + " target_version: str\n", + ") -> list[dict]:\n", + " \"\"\"Use the `uuid` in the change list to add in other attributes not given in change list.\"\"\"\n", + " source_flow_dict = get_elem_flow_dict(source_version)\n", + " target_flow_dict = get_elem_flow_dict(target_version)\n", + " \n", + " for change in changes:\n", + " if \"formula\" in change[\"source\"]:\n", + " del change[\"source\"][\"formula\"]\n", + " if \"formula\" in change[\"target\"]:\n", + " del change[\"target\"][\"formula\"]\n", + "\n", + " \n", + " change['source'].update(source_flow_dict[change['source']['uuid']])\n", + " del change['source']['uuid']\n", + " change['target'].update(target_flow_dict[change['target']['uuid']])\n", + " del change['target']['uuid']\n", + "\n", + " attributes = \", \".join([\n", + " key \n", + " for key, value in change['target'].items() \n", + " if key in change['source'] \n", + " and change['source'][key] != value\n", + " ])\n", + " comment = add_comment(\n", + " change.get(\"comment\"),\n", + " f\"Changed {attributes} from {source_version} to {target_version}.\"\n", + " )\n", + " change['comment'] = comment\n", + " change[\"source_version\"] = f\"ecoinvent-{source_version}-biosphere\"\n", + " change[\"target_version\"] = f\"ecoinvent-{target_version}-biosphere\"\n", + "\n", + " return changes" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "a7ba6cba-eedb-4502-befc-c4282e9b71a0", + "metadata": {}, + "outputs": [], + "source": [ + "def get_filtered_rd_changes(from_v: str, to_v: str) -> list[dict]:\n", + " \"\"\"Return a filtered list of biosphere changes where the name or uuid changed\"\"\"\n", + " raw = registry.get_file(f'ecoinvent-{from_v}-biosphere-ecoinvent-{to_v}-biosphere')\n", + " if 'replace' in raw:\n", + " data = raw['replace']\n", + " elif 'update' in raw:\n", + " data = raw['update']\n", + " else:\n", + " print(\"No update changes found\")\n", + " return []\n", + " data = [\n", + " obj\n", + " for obj in data\n", + " if 'name' in obj['target']\n", + " or obj['target']['uuid'] != obj['source']['uuid']\n", + " ]\n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "29185ef0-1fea-4b21-a4bd-3c8e18d2bc42", + "metadata": {}, + "outputs": [], + "source": [ + "def remove_only_synonyms_change(data: list[dict]) -> list[dict]:\n", + " good = []\n", + "\n", + " for line in data:\n", + " source = {k: v for k, v in line['source'].items() if k != \"synonyms\"}\n", + " target = {k: v for k, v in line['target'].items() if k != \"synonyms\"}\n", + " if source != target:\n", + " good.append(line)\n", + "\n", + " return good" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "b900b65d-6af7-4e86-8efd-91675942e248", + "metadata": {}, + "outputs": [], + "source": [ + "def apply_forward_change(data: list[dict], other: list[dict]) -> list[dict]:\n", + " \"\"\"Apply additional changes to get transitive change set.\"\"\"\n", + " other_mapping = {obj['source']['identifier']: obj for obj in other}\n", + " \n", + " for obj in data:\n", + " try:\n", + " transitive = other_mapping[obj['target']['identifier']]\n", + " obj['target'] = transitive['target']\n", + " if transitive.get(\"comment\"):\n", + " obj['comment'] = add_comment(obj.get(\"comment\"), addition=transitive[\"comment\"])\n", + " obj[\"target_version\"] = transitive[\"target_version\"]\n", + "\n", + " if \"conversion_factor\" in transitive:\n", + " obj[\"conversion_factor\"] = obj.get(\"conversion_factor\", 1.) * transitive[\"conversion_factor\"]\n", + " \n", + " logger.debug(\"Mapping change: %s\", obj)\n", + " except KeyError:\n", + " continue\n", + "\n", + " input_uuids = {obj['source'].get('identifier', None) for obj in data}\n", + " extra = [obj for obj in other if obj['source']['identifier'] not in input_uuids]\n", + " \n", + " return data + remove_only_synonyms_change(extra)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "34d4eb99-3dcf-4f0f-9121-c93b68ce0c85", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_transitive_datapackage(data: list[dict], source_id: str, end_target: str) -> rn.Datapackage:\n", + " dp = rn.Datapackage(\n", + " name=f\"ecoinvent-{source_id}-biosphere-ecoinvent-{end_target}-biosphere-transitive\",\n", + " source_id=f\"ecoinvent-{source_id}-biosphere\",\n", + " target_id=f\"ecoinvent-{end_target}-biosphere\",\n", + " description=f\"Transitive ecoinvent elementary flow correspondence from {source_id} to {end_target}\",\n", + " contributors=[{\"title\": \"Chris Mutel\", \"roles\": [\"author\"], \"path\": \"https://chris.mutel.org\"}],\n", + " mapping_source=rn.MappingConstants.ECOSPOLD2_BIO_FLOWMAPPER,\n", + " mapping_target=rn.MappingConstants.ECOSPOLD2_BIO_FLOWMAPPER,\n", + " version=\"1.0\",\n", + " )\n", + " dp.add_data(verb=\"update\", data=data)\n", + " filename = f\"ecoinvent-{source_id}-biosphere-ecoinvent-{end_target}-biosphere-transitive.json\"\n", + " dp.to_json(filename)\n", + " registry.add_file(filename, replace=True)\n", + " return dp" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "9b5d3432-dc49-4089-a416-94a40f070de7", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_datapackage(data: list[dict], source_id: str, target_id: str) -> rn.Datapackage:\n", + " dp = rn.Datapackage(\n", + " name=f\"ecoinvent-{source_id}-biosphere-ecoinvent-{target_id}-biosphere\",\n", + " source_id=f\"ecoinvent-{source_id}-biosphere\",\n", + " target_id=f\"ecoinvent-{target_id}-biosphere\",\n", + " description=f\"ecoinvent elementary flow correspondence from {source_id} to {target_id}\",\n", + " contributors=[{\"title\": \"Chris Mutel\", \"roles\": [\"author\"], \"path\": \"https://chris.mutel.org\"}],\n", + " mapping_source=rn.MappingConstants.ECOSPOLD2_BIO_FLOWMAPPER,\n", + " mapping_target=rn.MappingConstants.ECOSPOLD2_BIO_FLOWMAPPER,\n", + " version=\"1.0\",\n", + " )\n", + " dp.add_data(verb=\"update\", data=data)\n", + " filename = f\"ecoinvent-{source_id}-biosphere-ecoinvent-{target_id}-biosphere.json\"\n", + " dp.to_json(filename)\n", + " registry.add_file(filename, replace=True)\n", + " return dp" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "46265815-4d3b-442a-ad19-37c98d94d04d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Flowmapper-standard-units-harmonization',\n", + " 'SimaPro-2025-ecoinvent-3.12-context',\n", + " 'SimaPro-9-ecoinvent-3.8-biosphere',\n", + " 'SimaPro-9-ecoinvent-3.8-biosphere-manual-matches',\n", + " 'SimaPro-9-ecoinvent-3.9-biosphere',\n", + " 'SimaPro-9-ecoinvent-3.9-biosphere-manual-matches',\n", + " 'agribalyse-3.1.1-biosphere-ecoinvent-3.8-biosphere',\n", + " 'agribalyse-3.1.1-delete-aggregated-ecoinvent',\n", + " 'agribalyse-3.1.1-ecoinvent-3.10-biosphere-manual-matches',\n", + " 'agribalyse-3.1.1-restore-simapro-ecoinvent-names',\n", + " 'agrifootprint-2022-delete-aggregated-ecoinvent',\n", + " 'agrifootprint-2022-ecoinvent-3.10-biosphere',\n", + " 'agrifootprint-2022-ecoinvent-3.8-biosphere',\n", + " 'agrifootprint-2022-restore-simapro-ecoinvent-names',\n", + " 'ecoinvent-2.2-biosphere-context-ecoinvent-3.0-biosphere-context',\n", + " 'ecoinvent-2.2-biosphere-ecoinvent-3.0-biosphere',\n", + " 'ecoinvent-2.2-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.01-biosphere-ecoinvent-3.1-biosphere',\n", + " 'ecoinvent-3.01-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.1-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.10.1-biosphere-EF-3.1-biosphere',\n", + " 'ecoinvent-3.10.1-biosphere-ecoinvent-3.11-biosphere',\n", + " 'ecoinvent-3.10.1-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.10.1-cutoff-ecoinvent-3.11-cutoff',\n", + " 'ecoinvent-3.11-biosphere-EF-3.1-biosphere',\n", + " 'ecoinvent-3.11-cutoff-ecoinvent-3.12-cutoff',\n", + " 'ecoinvent-3.2-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.3-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.3-biosphere-ecoinvent-3.4-biosphere',\n", + " 'ecoinvent-3.4-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.4-biosphere-ecoinvent-3.5-biosphere',\n", + " 'ecoinvent-3.5-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.5-biosphere-ecoinvent-3.6-biosphere',\n", + " 'ecoinvent-3.6-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.6-biosphere-ecoinvent-3.7-biosphere',\n", + " 'ecoinvent-3.7-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.7-biosphere-ecoinvent-3.8-biosphere',\n", + " 'ecoinvent-3.7.1-cutoff-ecoinvent-3.8-cutoff',\n", + " 'ecoinvent-3.8-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.8-biosphere-ecoinvent-3.9-biosphere',\n", + " 'ecoinvent-3.8-cutoff-ecoinvent-3.9-cutoff',\n", + " 'ecoinvent-3.9.1-biosphere-EF-3.1-biosphere',\n", + " 'ecoinvent-3.9.1-biosphere-ecoinvent-3.10-biosphere',\n", + " 'ecoinvent-3.9.1-biosphere-ecoinvent-3.12-biosphere-transitive',\n", + " 'ecoinvent-3.9.1-cutoff-ecoinvent-3.10-cutoff',\n", + " 'generic-brightway-unit-conversions',\n", + " 'generic-brightway-units-normalization',\n", + " 'simapro-9-ecoinvent-3-context',\n", + " 'simapro-9-ecoinvent-3-water-slash-m3',\n", + " 'simapro-ecoinvent-3.10-cutoff',\n", + " 'simapro-ecoinvent-3.5-apos',\n", + " 'simapro-ecoinvent-3.5-consequential',\n", + " 'simapro-ecoinvent-3.5-cutoff',\n", + " 'simapro-ecoinvent-3.8-cutoff',\n", + " 'simapro-ecoinvent-3.9.1-cutoff']" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sorted(list(registry))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "4656eba5-4957-478d-a4c8-b853e0da8677", + "metadata": {}, + "outputs": [], + "source": [ + "previous = None\n", + "config = [\n", + " {\n", + " \"rd_source\": \"3.10.1\",\n", + " \"ei_source\": \"3.10.1\",\n", + " \"rd_target\": \"3.11\",\n", + " \"ei_target\": \"3.11\",\n", + " \"supplement\": True,\n", + " },\n", + " {\n", + " \"rd_source\": \"3.9.1\",\n", + " \"ei_source\": \"3.9.1\",\n", + " \"rd_target\": \"3.10\",\n", + " \"ei_target\": \"3.10.1\",\n", + " \"supplement\": True,\n", + " },\n", + " {\n", + " \"rd_source\": \"3.8\",\n", + " \"ei_source\": \"3.8\",\n", + " \"rd_target\": \"3.9\",\n", + " \"ei_target\": \"3.9.1\",\n", + " \"supplement\": True,\n", + " },\n", + " {\n", + " \"ei_source\": \"3.7\",\n", + " \"ei_target\": \"3.8\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.6\",\n", + " \"ei_target\": \"3.7\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.5\",\n", + " \"ei_target\": \"3.6\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.4\",\n", + " \"ei_target\": \"3.5\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.3\",\n", + " \"ei_target\": \"3.4\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.2\",\n", + " \"ei_target\": \"3.3\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.1\",\n", + " \"ei_target\": \"3.2\",\n", + " },\n", + " {\n", + " \"ei_source\": \"3.01\",\n", + " \"ei_target\": \"3.1\",\n", + " },\n", + "]\n", + "end_target = \"3.12\"" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "af525429-ba7a-42db-8509-ee279cfd70c8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████████| 11/11 [00:06<00:00, 1.78it/s]\n" + ] + } + ], + "source": [ + "for line in tqdm(config):\n", + " if line.get(\"supplement\"):\n", + " data = supplement_changes(\n", + " get_filtered_rd_changes(line[\"rd_source\"], line[\"rd_target\"]), \n", + " line[\"ei_source\"], \n", + " line[\"ei_target\"],\n", + " )\n", + " else:\n", + " data = generate_changes(line[\"ei_source\"], line[\"ei_target\"])\n", + "\n", + " if not line.get(\"supplement\") and data:\n", + " generate_datapackage(deepcopy(data), line[\"ei_source\"], line[\"ei_target\"])\n", + " \n", + " if previous is not None:\n", + " data = apply_forward_change(data, previous)\n", + "\n", + " generate_transitive_datapackage(deepcopy(data), line[\"ei_source\"], end_target)\n", + " previous = data" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c1d13a46-ddc8-4420-8cc9-0dd42dbb5742", + "metadata": {}, + "outputs": [], + "source": [ + "data_22 = registry.get_file('ecoinvent-2.2-biosphere-ecoinvent-3.0-biosphere')['replace']\n", + "\n", + "data_301 = defaultdict(list)\n", + "\n", + "for obj in get_elem_flow_data(\"3.01\"):\n", + " data_301[obj['name']].append(obj)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "8d94bc46-dc25-41d3-aef2-caf3b7ad1fb9", + "metadata": {}, + "outputs": [], + "source": [ + "changes = []\n", + "\n", + "for line in data_22:\n", + " s_name, t_name = line['source']['name'], line['target']['name']\n", + " if s_name == t_name:\n", + " continue\n", + " for obj in data_301.get(t_name, []):\n", + " source = {\n", + " \"name\": s_name,\n", + " \"context\": obj[\"context\"],\n", + " }\n", + " changes.append({\n", + " \"source\": source,\n", + " \"target\": obj,\n", + " \"comment\": \"Name change from ecoinvent 2.2 to 3.01\",\n", + " \"source_version\": \"ecoinvent-2.2-biosphere\",\n", + " \"target_version\": \"ecoinvent-3.01-biosphere\",\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "e7adaffb-84ba-4359-a1c9-3af052772445", + "metadata": {}, + "outputs": [], + "source": [ + "data = apply_forward_change(changes, previous)\n", + "\n", + "dp = generate_transitive_datapackage(data, \"2.2\", end_target)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d325e762-f400-45fb-b432-754321e44b9d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/README.md b/README.md index 044f0f0..d6c5bab 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,46 @@ [pre-commit]: https://github.com/pre-commit/pre-commit [black]: https://github.com/psf/black +## Flowmapper + +This is a tool to automate matching elementary flow lists in life cycle assessment. It can map +different lists against each other and generate usable transformation files for those mappings. + +It works by starting with the assumption that there is a core underlying ontology for elementary +flows, regardless of the expression of that ontology in difference data formats. Here is the core +ontology that `flowmapper` uses: + +* name: str, the canonical name used to identify a substance, e.g. "1,4-Butanediol" +* identifier: str, or complex type with a string representation, the unique string used to identify the flow, unit, and context combination, e.g. "38a622c6-f086-4763-a952-7c6b3b1c42ba" +* context: tuple[str], a hierarchical organization into environmental compartments, e.g. `("air", "urban air close to ground")` +* unit: str, or complex type with a string representation, e.g. "kg" +* sector-specific labels: str, or complex type with a string representation, a set of additional fields which can help identify or further specify a flow, e.g. CAS number 000110-63-4 +* synonyms: list[str], a list of alternative unique names for a substance, e.g. `["Butylene glycol", "butane-1,4-diol"]`. Synonyms should not overlap. + +Flowmapper **assumes that the source and target lists are given in this format**; it comes with or plays well with conversion software for data formats like ecospold, FEDEFL, and SimaPro CSV. + +Chemical formulas are not currently used - they can be expressed in too many different ways, and haven't proven useful for matching. + +Matching across the two lists is defined by a set of strategy functions; matching can take some or +all ontological elements into account. Due to the sometimes chaotic nature of the input data, +matching usually needs to be customized for the specific source and target lists. + +For example, a matching strategy could say that two flows are the same if their name and context are equal, or if their CAS number and context are equal. Units are rarely compared directly; instead, after a match is found we check that the units have [the same dimension](https://en.wikipedia.org/wiki/Dimensional_analysis), and apply unit conversions using [pint](https://pint.readthedocs.io/en/stable/) if necessary. + +This library does not generate partial matches (i.e. "1,4-Butanediol" is always the same as "butane-1,4-diol", but the separate contexts would need to be matched afterwards). Partial matches could be used to reduce the size of the matching file, as one could store name matches in one partial match and context matches in another partial match, but we prefer the explicit enumeration of all matches of flows present in the source and target lists. + +Matching is usually done by checking for equality. Instead of forcing conversion and then testing, +flowmapper comes with custom classes for these ontological elements which allow for a flexible +definition of equality, and the testing of logical relationships. For example, you can do this: + +```python +from flowmapper import CAS +first = CAS(" 007440-05-3") +second = CAS("7440-05-3") +first == second +>>> True +``` + ## Installation You can install _flowmapper_ via [pip] from [PyPI]: diff --git a/docs/conf.py b/docs/conf.py index a526103..98f9d00 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,5 @@ """Sphinx configuration.""" + import importlib.metadata import os import shutil @@ -37,9 +38,9 @@ "myst_parser", ] source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", - } + ".rst": "restructuredtext", + ".md": "markdown", +} autodoc_typehints = "description" html_theme = "furo" diff --git a/ecoinvent-3.10-biosphere-simapro-2024-biosphere.json b/ecoinvent-3.10-biosphere-simapro-2024-biosphere.json new file mode 100644 index 0000000..ad24edc --- /dev/null +++ b/ecoinvent-3.10-biosphere-simapro-2024-biosphere.json @@ -0,0 +1,1134 @@ +{ + "name": "ecoinvent-3.10-biosphere-simapro-2024-biosphere", + "description": "Manual SimaPro to ecoinvent name matches without conversion factors", + "contributors": [ + { + "title": "Chris Mutel", + "roles": [ + "author" + ], + "path": "https://chris.mutel.org" + } + ], + "created": "2025-11-16T22:04:33.433871+00:00", + "version": "1.0", + "licenses": [ + { + "name": "CC-BY-4.0", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "title": "Creative Commons Attribution 4.0 International" + } + ], + "graph_context": [ + "edges" + ], + "mapping": { + "source": { + "expression language": "XPath", + "labels": { + "name": "//*:elementaryExchange/*:name/text()", + "cas_number": "//*:elementaryExchange/@casNumber", + "unit": "//*:elementaryExchange/*:unitName/text()", + "identifier": "//*:elementaryExchange/@elementaryExchangeId", + "context": [ + "//*:elementaryExchange/*:compartment/*:compartment/text()", + "//*:elementaryExchange/*:compartment/*:subcompartment/text()" + ], + "synonyms": "//*:elementaryExchange/*:synonym/text()" + } + }, + "target": { + "expression language": "like JSONPath", + "labels": { + "identifier": "Process[*].\"Process identifier\".text", + "name": "Process[*].Products[*].text[0]", + "platform_id": "Process[*].\"Platform Identifier\"", + "unit": [ + "[\"Emissions to air/\", Process[*].\"Emissions to air\".[2]]", + "[\"Emissions to soil/\", Process[*].\"Emissions to soil\".[2]]", + "[\"Emissions to water/\", Process[*].\"Emissions to water\".[2]]", + "[\"Resources/\", Process[*].\"Resources\".[2]]" + ], + "context": [ + "[\"Emissions to air/\", Process[*].\"Emissions to air\".[1]]", + "[\"Emissions to soil/\", Process[*].\"Emissions to soil\".[1]]", + "[\"Emissions to water/\", Process[*].\"Emissions to water\".[1]]", + "[\"Resources/\", Process[*].\"Resources\".[1]]" + ] + } + } + }, + "source_id": "simapro-2024-biosphere", + "target_id": "ecoinvent-3.10-biosphere", + "update": [ + { + "source": { + "name": "Parathion, methyl" + }, + "target": { + "name": "Methyl parathion" + } + }, + { + "source": { + "name": "Thiocyanic acid (-1 ion)" + }, + "target": { + "name": "Thiocyanate" + } + }, + { + "source": { + "name": "Quizalofop ethyl ester" + }, + "target": { + "name": "Quizalofop-ethyl" + } + }, + { + "source": { + "name": "Prothioconazol" + }, + "target": { + "name": "Prothioconazole" + } + }, + { + "source": { + "name": "Pyraclostrobin (prop)" + }, + "target": { + "name": "Pyraclostrobin" + } + }, + { + "source": { + "name": "Monosodium acid methanearsonate" + }, + "target": { + "name": "MSMA" + } + }, + { + "source": { + "name": "Carbamic acid, [(dibutylamino)thio]methyl-, 2,3-dihydro-2,2-dimethyl-7-benzofuranyl ester" + }, + "target": { + "name": "Carbosulfan" + } + }, + { + "source": { + "name": "Benzene, 1-methyl-2-nitro-" + }, + "target": { + "name": "o-Nitrotoluene" + } + }, + { + "source": { + "name": "Alkane (unspecified)" + }, + "target": { + "name": "Hydrocarbons, aliphatic, alkanes, unspecified" + } + }, + { + "source": { + "name": "AOX (Adsorbable Organic Halogens)" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "AOX, Adsorbable Organic Halogen as Cl" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "BOD5 (Biological Oxygen Demand)" + }, + "target": { + "name": "BOD5, Biological Oxygen Demand" + } + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand)" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + } + }, + { + "source": { + "name": "Wood, unspecified, standing/m3" + }, + "target": { + "name": "Wood, unspecified, standing" + } + }, + { + "source": { + "name": "Particulates, < 2.5 um" + }, + "target": { + "name": "Particulate Matter, < 2.5 um" + } + }, + { + "source": { + "name": "Particulates, > 10 um" + }, + "target": { + "name": "Particulate Matter, > 10 um" + } + }, + { + "source": { + "name": "Particulates, > 2.5 um, and < 10um" + }, + "target": { + "name": "Particulate Matter, > 2.5 um and < 10um" + } + }, + { + "source": { + "name": "Sand" + }, + "target": { + "name": "Sand, unspecified" + } + }, + { + "source": { + "name": "Potassium chloride" + }, + "target": { + "name": "Sylvite" + } + }, + { + "source": { + "name": "Sodium tetrahydroborate" + }, + "target": { + "name": "Sodium tetrahydridoborate" + } + }, + { + "source": { + "name": "Toluene, 2-chloro-" + }, + "target": { + "name": "o-Chlorotoluene" + } + }, + { + "source": { + "name": "Pentane, 2,2,4-trimethyl-" + }, + "target": { + "name": "2,2,4-Trimethylpentane" + } + }, + { + "source": { + "name": "Dioxin, 2,3,7,8 Tetrachlorodibenzo-p-" + }, + "target": { + "name": "Dioxins, measured as 2,3,7,8-tetrachlorodibenzo-p-dioxin" + } + }, + { + "source": { + "name": "Discarded fish, demersal" + }, + "target": { + "name": "Discarded fish, demersal, to ocean" + } + }, + { + "source": { + "name": "Methane, tetrachloro-, CFC-10" + }, + "target": { + "name": "Carbon tetrachloride" + } + }, + { + "source": { + "name": "Methane, tetrafluoro-, CFC-14" + }, + "target": { + "name": "Tetrafluoromethane" + } + }, + { + "source": { + "name": "Metolachlor, (S)" + }, + "target": { + "name": "Metolachlor" + } + }, + { + "source": { + "name": "Methane, chlorofluoro-, HCFC-31" + }, + "target": { + "name": "Chloro-fluoromethane" + } + }, + { + "source": { + "name": "Metam-sodium dihydrate" + }, + "target": { + "name": "Metam-sodium" + } + }, + { + "source": { + "name": "Gas, natural, 36 MJ per m3" + }, + "target": { + "name": "Gas, natural, in ground" + } + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining, 36 MJ per m3" + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining" + } + }, + { + "source": { + "name": "Discarded fish, pelagic" + }, + "target": { + "name": "Discarded fish, pelagic, to ocean" + } + }, + { + "source": { + "name": "Dipropylthiocarbamic acid S-ethyl ester" + }, + "target": { + "name": "EPTC" + } + }, + { + "source": { + "name": "Oxydemeton methyl" + }, + "target": { + "name": "Oxydemeton-methyl" + } + }, + { + "source": { + "name": "Thiazole, 2-(thiocyanatemethylthio)benzo-" + }, + "target": { + "name": "TCMTB" + } + }, + { + "source": { + "name": "Tri-allate" + }, + "target": { + "name": "Triallate" + } + }, + { + "source": { + "name": "Cesium (I)" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Cesium" + }, + "target": { + "name": "Caesium" + } + }, + { + "source": { + "name": "Dimethyl formamide" + }, + "target": { + "name": "N,N-Dimethylformamide" + } + }, + { + "source": { + "name": "Methane, land transformation" + }, + "target": { + "name": "Methane, from soil or biomass stock" + } + }, + { + "source": { + "name": "Carbon dioxide, land transformation" + }, + "target": { + "name": "Carbon dioxide, from soil or biomass stock" + } + }, + { + "source": { + "name": "Carbon monoxide, land transformation" + }, + "target": { + "name": "Carbon monoxide, from soil or biomass stock" + } + }, + { + "source": { + "name": "Nitrogen, atmospheric" + }, + "target": { + "name": "Nitrogen" + } + }, + { + "source": { + "name": "Butyric acid, 4-(2,4-dichlorophenoxy)-" + }, + "target": { + "name": "2,4-DB" + } + }, + { + "source": { + "name": "Benzo(a)anthracene" + }, + "target": { + "name": "Benz(a)anthracene" + } + }, + { + "source": { + "name": "Oil, crude, 43.4 MJ per kg" + }, + "target": { + "name": "Oil, crude" + } + }, + { + "source": { + "name": "Argon-40/kg" + }, + "target": { + "name": "Argon" + } + }, + { + "source": { + "name": "1-Butanol" + }, + "target": { + "name": "Butanol" + } + }, + { + "source": { + "name": "Metaldehyde (tetramer)" + }, + "target": { + "name": "Metaldehyde" + } + }, + { + "source": { + "name": "Roundup" + }, + "target": { + "name": "Glyphosate" + } + }, + { + "source": { + "name": "Transformation, from pasture and meadow, organic" + }, + "target": { + "name": "Transformation, from pasture, man made, extensive" + } + }, + { + "source": { + "name": "Transformation, to pasture and meadow, organic" + }, + "target": { + "name": "Transformation, to pasture, man made, extensive" + } + }, + { + "source": { + "name": "Transformation, from arable, organic" + }, + "target": { + "name": "Transformation, from arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to arable, organic" + }, + "target": { + "name": "Transformation, to arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to industrial area, built up" + }, + "target": { + "name": "Transformation, to industrial area", + "context": [ + "natural resource", + "land" + ] + } + }, + { + "source": { + "name": "Transformation, from agriculture" + }, + "target": { + "name": "Transformation, from annual crop" + } + }, + { + "source": { + "name": "Transformation, from annual crop, non-irrigated, fallow" + }, + "target": { + "name": "Transformation, from pasture, man made" + } + }, + { + "source": { + "name": "Transformation, from forest, intensive, clear-cutting" + }, + "target": { + "name": "Transformation, from forest, intensive" + } + }, + { + "source": { + "name": "Transformation, from forest, used" + }, + "target": { + "name": "Transformation, from forest, intensive" + } + }, + { + "source": { + "name": "Transformation, from grassland" + }, + "target": { + "name": "Transformation, from grassland, natural (non-use)" + } + }, + { + "source": { + "name": "Transformation, from grassland/pasture/meadow" + }, + "target": { + "name": "Transformation, from pasture, man made" + } + }, + { + "source": { + "name": "Transformation, from industrial area, benthos" + }, + "target": { + "name": "Transformation, from seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, from industrial area, built up" + }, + "target": { + "name": "Transformation, from industrial area" + } + }, + { + "source": { + "name": "Transformation, from industrial area, vegetation" + }, + "target": { + "name": "Transformation, from industrial area" + } + }, + { + "source": { + "name": "Transformation, from permanent crop, fruit" + }, + "target": { + "name": "Transformation, from permanent crop, irrigated" + } + }, + { + "source": { + "name": "Transformation, from tropical rain forest" + }, + "target": { + "name": "Transformation, from forest, extensive" + } + }, + { + "source": { + "name": "Transformation, from unspecified, used" + }, + "target": { + "name": "Transformation, from unspecified" + } + }, + { + "source": { + "name": "Transformation, to agriculture" + }, + "target": { + "name": "Transformation, to annual crop" + } + }, + { + "source": { + "name": "Transformation, to annual crop, fallow" + }, + "target": { + "name": "Transformation, to arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to annual crop, non-irrigated, fallow" + }, + "target": { + "name": "Transformation, to annual crop, non-irrigated, extensive" + } + }, + { + "source": { + "name": "Transformation, to dump site, benthos" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, clear-cutting" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, normal" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, short-cycle" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, used" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to grassland/pasture/meadow" + }, + "target": { + "name": "Transformation, to pasture, man made" + } + }, + { + "source": { + "name": "Transformation, to industrial area, benthos" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to industrial area, vegetation" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to permanent crop, fruit, intensive" + }, + "target": { + "name": "Transformation, to permanent crop, irrigated" + } + }, + { + "source": { + "name": "Transformation, to sea and ocean" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to traffic area, road embankment" + }, + "target": { + "name": "Transformation, to traffic area, road network" + } + }, + { + "source": { + "name": "Transformation, to unspecified, used" + }, + "target": { + "name": "Transformation, to unspecified" + } + }, + { + "source": { + "name": "Transformation, to urban/industrial fallow" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to water bodies, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Transformation, to water courses, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Transformation, to lakes, artificial" + }, + "target": { + "name": "Transformation, to lake, artificial" + } + }, + { + "source": { + "name": "Transformation, to rivers, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Occupation, lakes, artificial" + }, + "target": { + "name": "Occupation, lake, artificial" + } + }, + { + "source": { + "name": "Occupation, rivers, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, water bodies, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, agriculture" + }, + "target": { + "name": "Occupation, annual crop" + } + }, + { + "source": { + "name": "Occupation, dump site, benthos" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, forest, intensive, normal" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, forest, intensive, short-cycle" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, forest, used" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, grassland/pasture/meadow" + }, + "target": { + "name": "Occupation, pasture, man made" + } + }, + { + "source": { + "name": "Occupation, industrial area, benthos" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, industrial area, built up" + }, + "target": { + "name": "Occupation, industrial area" + } + }, + { + "source": { + "name": "Occupation, industrial area, vegetation" + }, + "target": { + "name": "Occupation, industrial area" + } + }, + { + "source": { + "name": "Occupation, permanent crop, fruit, intensive" + }, + "target": { + "name": "Occupation, permanent crop, irrigated" + } + }, + { + "source": { + "name": "Occupation, sea and ocean" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, sea and ocean" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, traffic area" + }, + "target": { + "name": "Occupation, traffic area, road network" + } + }, + { + "source": { + "name": "Occupation, traffic area, road embankment" + }, + "target": { + "name": "Occupation, traffic area, road network" + } + }, + { + "source": { + "name": "Occupation, unspecified, used" + }, + "target": { + "name": "Occupation, unspecified" + } + }, + { + "source": { + "name": "Occupation, water bodies, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, water courses, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, wetland" + }, + "target": { + "name": "Occupation, inland waterbody, unspecified" + } + }, + { + "source": { + "name": "Bauxite" + }, + "target": { + "name": "Gangue" + } + }, + { + "source": { + "name": "Copper ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 0.38%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Pb 0.014%, in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Coal, 18 MJ per kg" + }, + "target": { + "name": "Coal, hard" + } + }, + { + "source": { + "name": "Coal, brown, 10 MJ per kg" + }, + "target": { + "name": "Coal, brown" + } + }, + { + "source": { + "name": "Coal, brown, 8 MJ per kg" + }, + "target": { + "name": "Coal, brown" + } + }, + { + "source": { + "name": "Crude oil" + }, + "target": { + "name": "Oil, crude" + } + }, + { + "source": { + "name": "Energy, from biomass" + }, + "target": { + "name": "Energy, gross calorific value, in biomass" + } + }, + { + "source": { + "name": "Energy, from coal" + }, + "target": { + "name": "Energy, gross calorific value, in biomass" + } + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining/m3" + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining", + "unit": "Sm3" + } + }, + { + "source": { + "name": "Silver, Ag 9.7E-4%, Au 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Zinc" + } + }, + { + "source": { + "name": "Lead, Pb 0.014%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, in ore" + }, + "target": { + "name": "Lead" + } + }, + { + "source": { + "name": "Nickel, Ni 2.3E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Nickel, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Platinum, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Gold, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Platinum, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Palladium, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Palladium, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Clay" + }, + "target": { + "name": "Clay, bentonite" + } + }, + { + "source": { + "name": "Rhodium, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Rhodium, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Rhodium" + } + } + ] +} \ No newline at end of file diff --git a/file_logger.py b/file_logger.py new file mode 100644 index 0000000..1025eaa --- /dev/null +++ b/file_logger.py @@ -0,0 +1,337 @@ +import logging +import structlog +from pathlib import Path +from typing import Optional + + +def configure_file_logger( + logger_name: str, + log_file_path: str | Path, + log_level: int = logging.INFO, + log_format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s", + encoding: str = "utf-8", +) -> structlog.BoundLogger: + """ + Configure a structlog logger to log messages only to a file. + + Args: + logger_name: Name of the logger + log_file_path: Path to the log file + log_level: Logging level (default: INFO) + log_format: Format string for log messages + encoding: File encoding (default: utf-8) + + Returns: + structlog.BoundLogger: Configured structlog logger + + Example: + >>> logger = configure_file_logger("my_app", "logs/app.log") + >>> logger.info("Application started") + >>> logger.error("An error occurred", error_code=500) + """ + # Convert path to Path object if it's a string + log_file_path = Path(log_file_path) + + # Create log directory if it doesn't exist + log_file_path.parent.mkdir(parents=True, exist_ok=True) + + # Get the standard library logger + std_logger = logging.getLogger(logger_name) + std_logger.setLevel(log_level) + + # Remove only FileHandler handlers to avoid duplicates while preserving other handlers + for handler in std_logger.handlers[:]: + if isinstance(handler, logging.FileHandler): + std_logger.removeHandler(handler) + + # Create a simple file handler + file_handler = logging.FileHandler( + filename=log_file_path, + encoding=encoding, + ) + + # Create formatter + formatter = logging.Formatter(log_format) + file_handler.setFormatter(formatter) + + # Add handler to logger + std_logger.addHandler(file_handler) + + # Prevent propagation to root logger to avoid console output + std_logger.propagate = False + + # Get the structlog logger + logger = structlog.get_logger(logger_name) + + return logger + + +def configure_structured_file_logger( + logger_name: str, + log_file_path: str | Path, + log_level: int = logging.INFO, + encoding: str = "utf-8", + include_timestamp: bool = True, + include_logger_name: bool = True, + include_level: bool = True, +) -> structlog.BoundLogger: + """ + Configure a structlog logger with structured logging to a file. + + Args: + logger_name: Name of the logger + log_file_path: Path to the log file + log_level: Logging level (default: INFO) + encoding: File encoding (default: utf-8) + include_timestamp: Whether to include timestamp in logs (default: True) + include_logger_name: Whether to include logger name in logs (default: True) + include_level: Whether to include log level in logs (default: True) + + Returns: + structlog.BoundLogger: Configured structlog logger with structured logging + + Example: + >>> logger = configure_structured_file_logger("my_app", "logs/app.json") + >>> logger.info("User logged in", user_id=123, ip="192.168.1.1") + """ + import json + + # Convert path to Path object if it's a string + log_file_path = Path(log_file_path) + + # Create log directory if it doesn't exist + log_file_path.parent.mkdir(parents=True, exist_ok=True) + + # Get the standard library logger + std_logger = logging.getLogger(logger_name) + std_logger.setLevel(log_level) + + # Remove only FileHandler handlers to avoid duplicates while preserving other handlers + for handler in std_logger.handlers[:]: + if isinstance(handler, logging.FileHandler): + std_logger.removeHandler(handler) + + # Create a simple file handler + file_handler = logging.FileHandler( + filename=log_file_path, + encoding=encoding, + ) + + # Create JSON formatter for structured logging + class JSONFormatter(logging.Formatter): + def format(self, record): + log_entry = { + "message": record.getMessage(), + } + + if include_timestamp: + log_entry["timestamp"] = self.formatTime(record) + + if include_logger_name: + log_entry["logger"] = record.name + + if include_level: + log_entry["level"] = record.levelname + + # Add any extra fields from structlog + if hasattr(record, "structlog"): + log_entry.update(record.structlog) + + return json.dumps(log_entry) + + formatter = JSONFormatter() + file_handler.setFormatter(formatter) + + # Add handler to logger + std_logger.addHandler(file_handler) + + # Prevent propagation to root logger to avoid console output + std_logger.propagate = False + + # Get the structlog logger + logger = structlog.get_logger(logger_name) + + return logger + + +def get_file_logger( + logger_name: str, + log_file_path: str | Path, + structured: bool = False, + **kwargs +) -> structlog.BoundLogger: + """ + Convenience function to get a file logger with either standard or structured logging. + + Args: + logger_name: Name of the logger + log_file_path: Path to the log file + structured: Whether to use structured (JSON) logging (default: False) + **kwargs: Additional arguments passed to the configuration function + + Returns: + structlog.BoundLogger: Configured structlog logger + + Example: + >>> # Standard logging + >>> logger = get_file_logger("app", "logs/app.log") + >>> + >>> # Structured logging + >>> logger = get_file_logger("app", "logs/app.json", structured=True) + """ + if structured: + return configure_structured_file_logger(logger_name, log_file_path, **kwargs) + else: + return configure_file_logger(logger_name, log_file_path, **kwargs) + + +def reset_logger_to_defaults(logger_name: str) -> structlog.BoundLogger: + """ + Reset a named structlog logger to its default configuration. + + This function removes all custom handlers and resets the logger to use + the default structlog configuration, which typically outputs to console. + + Args: + logger_name: Name of the logger to reset + + Returns: + structlog.BoundLogger: Reset structlog logger + + Example: + >>> # Configure a file logger + >>> logger = configure_file_logger("my_app", "logs/app.log") + >>> logger.info("This goes to file") + >>> + >>> # Reset to defaults (console output) + >>> logger = reset_logger_to_defaults("my_app") + >>> logger.info("This goes to console") + """ + # Get the standard library logger + std_logger = logging.getLogger(logger_name) + + # Remove all existing handlers + for handler in std_logger.handlers[:]: + std_logger.removeHandler(handler) + + # Reset logger level to default (NOTSET) + std_logger.setLevel(logging.NOTSET) + + # Re-enable propagation to parent loggers + std_logger.propagate = True + + # Get the structlog logger (this will use default structlog configuration) + logger = structlog.get_logger(logger_name) + + return logger + + +def reset_all_loggers_to_defaults() -> None: + """ + Reset all loggers to their default configuration. + + This function removes all custom handlers from all loggers and resets + them to use the default structlog configuration. + + Example: + >>> # Configure multiple file loggers + >>> logger1 = configure_file_logger("app1", "logs/app1.log") + >>> logger2 = configure_file_logger("app2", "logs/app2.log") + >>> + >>> # Reset all loggers to defaults + >>> reset_all_loggers_to_defaults() + >>> + >>> # Now all loggers will output to console by default + >>> logger1.info("This goes to console") + >>> logger2.info("This also goes to console") + """ + # Get all existing loggers + loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] + + # Also include the root logger + loggers.append(logging.getLogger()) + + for logger in loggers: + # Remove all existing handlers + for handler in logger.handlers[:]: + logger.removeHandler(handler) + + # Reset logger level to default + logger.setLevel(logging.NOTSET) + + # Re-enable propagation + logger.propagate = True + + +def get_logger_info(logger_name: str) -> dict: + """ + Get information about a logger's current configuration. + + Args: + logger_name: Name of the logger to inspect + + Returns: + dict: Information about the logger's configuration + + Example: + >>> logger = configure_file_logger("my_app", "logs/app.log") + >>> info = get_logger_info("my_app") + >>> print(info) + >>> # Output: {'name': 'my_app', 'level': 20, 'handlers': 1, 'propagate': False} + """ + std_logger = logging.getLogger(logger_name) + + return { + "name": logger_name, + "level": std_logger.level, + "handlers": len(std_logger.handlers), + "propagate": std_logger.propagate, + "handler_types": [type(handler).__name__ for handler in std_logger.handlers], + } + + +# Example usage and testing +if __name__ == "__main__": + # Example 1: Standard file logging + logger1 = configure_file_logger("test_app", "logs/test.log") + logger1.info("This is a test message") + logger1.error("This is an error message", error_code=500) + + # Example 2: Structured file logging + logger2 = configure_structured_file_logger("test_app_structured", "logs/test.json") + logger2.info("User action", user_id=123, action="login", ip="192.168.1.1") + logger2.error("Database error", error_code=500, table="users", query="SELECT *") + + # Example 3: Using convenience function + logger3 = get_file_logger("convenience_app", "logs/convenience.log") + logger3.info("Using convenience function") + + logger4 = get_file_logger("convenience_structured", "logs/convenience.json", structured=True) + logger4.info("Structured logging with convenience", event="test", data={"key": "value"}) + + # Example 4: Demonstrating reset functionality + print("\n=== Testing Reset Functionality ===") + + # Show logger info before reset + print("Before reset:") + print(f"test_app logger info: {get_logger_info('test_app')}") + + # Reset specific logger + reset_logger = reset_logger_to_defaults("test_app") + reset_logger.info("This message goes to console (after reset)") + + # Show logger info after reset + print("After reset:") + print(f"test_app logger info: {get_logger_info('test_app')}") + + # Example 5: Reset all loggers + print("\n=== Resetting All Loggers ===") + reset_all_loggers_to_defaults() + + # All loggers now use default configuration + logger1.info("This also goes to console now") + logger2.info("This also goes to console now") + + print("\nLog files created successfully!") + print("Check the 'logs' directory for the generated log files.") + print("After reset, all loggers output to console by default.") \ No newline at end of file diff --git a/flowmapper/__init__.py b/flowmapper/__init__.py deleted file mode 100644 index a4865c0..0000000 --- a/flowmapper/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""flowmapper.""" - -from .flowmap import Flowmap - -__all__ = ( - "__version__", - "Flowmap" - # Add functions and variables you want exposed in `flowmapper.` namespace here -) - -__version__ = "0.0.0-9010" diff --git a/flowmapper/cas.py b/flowmapper/cas.py deleted file mode 100644 index 17a2867..0000000 --- a/flowmapper/cas.py +++ /dev/null @@ -1,41 +0,0 @@ -from functools import cached_property - - -class CAS: - """ - Class for CAS Registry Numbers® (CAS RN®) that accepts padded or non-padded strings - """ - def __init__(self, cas: str): - if not isinstance(cas, str) and cas is not None: - raise TypeError(f'cas should be a str, not {type(cas).__name__}') - else: - cas = '' if cas is None else cas - self.cas = cas.strip().lstrip('0') - self.digits = tuple(int(d) for d in self.cas[0:-1].replace('-', '')) - def __repr__(self): - return self.cas - def __eq__(self, other): - if not self.cas and not other.cas: - result = False - else: - result = self.cas == other.cas - return result - @cached_property - def check_digit_expected(self): - """ - Expected digit acording to https://www.cas.org/support/documentation/chemical-substances/checkdig algorithm - """ - result = sum([ - index * value - for index, value in enumerate(self.digits[::-1], start=1) - ]) % 10 - return result - @property - def check_digit_actual(self): - int(self.cas[-1]) - @property - def valid(self): - """ - True if check if CAS number is valid acording to https://www.cas.org/support/documentation/chemical-substances/checkdig algorithm - """ - return self.check_digit_actual == self.check_digit_expected diff --git a/flowmapper/cli.py b/flowmapper/cli.py deleted file mode 100644 index 262927b..0000000 --- a/flowmapper/cli.py +++ /dev/null @@ -1,94 +0,0 @@ -import typer -from typing_extensions import Annotated -from pathlib import Path -import logging -from typing import Optional -from typing_extensions import Annotated -import importlib.metadata -import logging -import json -from pathlib import Path -from enum import Enum -from randonneur import migrate_datasets -from .flow import Flow -from .utils import read_field_mapping, read_flowlist, read_migration_files -from .flowmap import Flowmap -try: - import tomllib -except ModuleNotFoundError: - import tomli as tomllib - -logger = logging.getLogger(__name__) - -app = typer.Typer() - -def version_callback(value: bool): - if value: - print(f"flowmapper, version {importlib.metadata.version('flowmapper')}") - raise typer.Exit() - -class OutputFormat(str, Enum): - all = 'all' - glad = 'glad' - randonneur = 'randonneur' - -@app.callback() -def main(version: Annotated[ - Optional[bool], - typer.Option("--version", callback=version_callback, is_eager=True), - ] = None,): - """ - Generate mappings between elementary flows lists - """ - -@app.command() -def map( - source: Annotated[Path, typer.Argument(help='Path to source flowlist')], - target: Annotated[Path, typer.Argument(help='Path to target flowlist')], - fields: Annotated[Path, typer.Option(help='Relationship between fields in source and target flowlists')], - output_dir: Annotated[Path, typer.Option(help='Directory to save mapping and diagnostics files')] = Path('.'), - format: Annotated[OutputFormat, typer.Option(help='Mapping file output format', case_sensitive=False)] = 'all', - transformations: Annotated[list[Path], typer.Option("--transformations", "-t", help='Randonneur data migration file with changes to be applied to source flows before matching')] = None, - unmatched_source: Annotated[bool, typer.Option(help='Write original source unmatched flows into separate file?')] = True, - unmatched_target: Annotated[bool, typer.Option(help='Write original target unmatched flows into separate file?')] = True, - matched_source: Annotated[bool, typer.Option(help='Write original source matched flows into separate file?')] = False, - matched_target: Annotated[bool, typer.Option(help='Write original target matched flows into separate file?')] = False, -): - """ - Generate mappings between elementary flows lists - """ - output_dir.mkdir(parents=True, exist_ok=True) - field_mapping = read_field_mapping(fields) - if transformations: - transformations = read_migration_files(*transformations) - source_flows = [Flow(flow, field_mapping['source'], transformations) for flow in read_flowlist(source)] - target_flows = [Flow(flow, field_mapping['target']) for flow in read_flowlist(target)] - - flowmap = Flowmap(source_flows, target_flows) - flowmap.statistics() - - stem = f'{source.stem}-{target.stem}' - - if matched_source: - with open(output_dir / f'{stem}-matched-source.json', 'w') as fs: - json.dump([flow.raw for flow in flowmap.matched_source], fs, indent=True) - - if unmatched_source: - with open(output_dir / f'{stem}-unmatched-source.json', 'w') as fs: - json.dump([flow.raw for flow in flowmap.unmatched_source], fs, indent=True) - - if matched_target: - with open(output_dir / f'{stem}-matched-target.json', 'w') as fs: - json.dump([flow.raw for flow in flowmap.matched_target], fs, indent=True) - - if unmatched_target: - with open(output_dir / f'{stem}-unmatched-target.json', 'w') as fs: - json.dump([flow.raw for flow in flowmap.unmatched_target], fs, indent=True) - - if format.value == 'randonneur': - flowmap.to_randonneur(output_dir / f'{stem}.json') - elif format.value == 'glad': - flowmap.to_glad(output_dir / f'{stem}.xlsx') - else: - flowmap.to_randonneur(output_dir / f'{stem}.json') - flowmap.to_glad(output_dir / f'{stem}.xlsx') diff --git a/flowmapper/constants.py b/flowmapper/constants.py deleted file mode 100644 index 06f36dc..0000000 --- a/flowmapper/constants.py +++ /dev/null @@ -1,77 +0,0 @@ -CONTEXT_MAPPING = { - 'air/high. pop.': 'air/urban air close to ground', - 'air/indoor': 'air/indoor', - 'air/low. pop.': 'air/non-urban air or from high stacks', - 'air/low. pop., long-term': 'air/low population density, long-term', - 'air/stratosphere + troposphere': 'air/lower stratosphere + upper troposphere', - 'emissions to air': 'air', - 'emissions to air/high. pop.': 'air/urban air close to ground', - 'emissions to air/low. pop.': 'air/non-urban air or from high stacks', - 'emissions to air/low. pop., long-term': 'air/low population density, long-term', - 'emissions to air/stratosphere + troposphere': 'air/lower stratosphere + upper troposphere', - 'emissions to soil': 'soil', - 'emissions to soil/agricultural': 'soil/agricultural', - 'emissions to soil/forestry': 'soil/forestry', - 'emissions to soil/industrial': 'soil/industrial', - 'water/groundwater, long-term': 'water/ground-, long-term', - 'water/river': 'water/surface water', - 'emissions to water': 'water', - 'emissions to water/groundwater': 'water/ground-', - 'emissions to water/groundwater, long-term': 'water/ground-, long-term', - 'emissions to water/lake': 'water/surface water', - 'emissions to water/ocean': 'water/ocean', - 'emissions to water/river': 'water/surface water', - 'emissions to water/river, long-term': 'water/surface water', - 'resources': 'natural resource/in ground', - 'resources/biotic': 'natural resource/biotic', - 'resources/in air': 'natural resource/in air', - 'resources/in ground': 'natural resource/in ground', - 'resources/in water': 'natural resource/in water', - 'resources/land': 'natural resource/land', - 'raw': 'natural resource/in ground', - 'raw/in water': 'natural resource/in water', - } - -UNITS_NORMALIZATION = { - "a": "year", # Common in LCA circles; could be confused with are - "bq": "becquerel", - "g": "gram", - "gj": "gigajoule", - "h": "hour", - "ha": "hectare", - "hr": "hour", - "kbq": "kilobecquerel", - "kg": "kilogram", - "kgkm": "kilogram_kilometer", - "km": "kilometer", - "kj": "kilojoule", - "kwh": "kilowatt hour", - "l": "litre", - "lu": "livestock unit", - "m": "meter", - "m*year": "meter_year", - "m2": "square_meter", - "m2*year": "square_meter_year", - "m2a": "square_meter_year", - "m2y": "square_meter_year", - "m3": "cubic_meter", - "m3*year": "cubic_meter_year", - "m3a": "cubic_meter_year", - "m3y": "cubic_meter_year", - "ma": "meter_year", - "metric ton*km": "ton_kilometer", - "mj": "megajoule", - "my": "meter_year", - "nm3": "cubic_meter", - "p": "unit", - "personkm": "person_kilometer", - "person*km": "person_kilometer", - "pkm": "person_kilometer", - "tkm": "ton_kilometer", - "vkm": "vehicle_kilometer", - 'kg sw': "kilogram_separative_work_unit", - 'km*year': "kilometer_year", - 'metric ton*km': "ton_kilometer", - 'person*km': "person_kilometer", - 'wh': 'watt_hour', -} diff --git a/flowmapper/context.py b/flowmapper/context.py deleted file mode 100644 index 9d55506..0000000 --- a/flowmapper/context.py +++ /dev/null @@ -1,55 +0,0 @@ -from dataclasses import asdict, dataclass, field -from .constants import CONTEXT_MAPPING - -import flowmapper.jsonpath as jp - -@dataclass -class Context: - value: str - raw_value: str = "" - raw_object: dict = field(default_factory=lambda: {}) - - @classmethod - def from_dict(cls, d, spec): - if not spec: - return Context(None) - else: - key = jp.root(spec) - value = cls.ensure_list(jp.extract(spec, d)) - result = Context( - value = cls.normalize_contexts(value), - raw_value = '/'.join(value), - raw_object = {key: d.get(key)}, - ) - return result - - def to_dict(self): - return asdict(self) - - def __eq__(self, other): - if self and other and isinstance(other, Context): - return self.value == other.value - elif self and other: - return self.value == other - else: - return False - - def __bool__(self): - return bool(self.value) - - def __hash__(self): - return hash(self.value) - - @staticmethod - def ensure_list(x): - return x.split('/') if isinstance(x, str) else x - - @staticmethod - def normalize_contexts(context: list[str]): - result = '/'.join([ - segment.lower().rstrip('/') - for segment in context - if segment and segment.lower() not in {'(unspecified)', 'unspecified'} - ]) - - return CONTEXT_MAPPING.get(result, result) \ No newline at end of file diff --git a/flowmapper/data/units.txt b/flowmapper/data/units.txt deleted file mode 100644 index 892f3b5..0000000 --- a/flowmapper/data/units.txt +++ /dev/null @@ -1,6 +0,0 @@ -@alias becquerel = bq -square_meter = meter ** 2 = m2 -cubic_meter = (meter ** 3) = m3 -square_meter_year = m2 * year = m2y = m2a -cubic_meter_year = m3 * year = m3y = m3a -standard_cubic_meter = (meter ** 3) = sm3 diff --git a/flowmapper/flow.py b/flowmapper/flow.py deleted file mode 100644 index 103b133..0000000 --- a/flowmapper/flow.py +++ /dev/null @@ -1,63 +0,0 @@ -import flowmapper.jsonpath as jp - -from .unit import Unit -from .cas import CAS -from .context import Context -from .utils import generate_flow_id, transform_flow, find_transformation -from .flowproperty import FlowProperty - - -class Flow: - def __init__( - self, - original, - fields=None, - transformations=None, - ): - transformation = find_transformation(original, transformations) - transformed = ( - transform_flow(original, transformation) if transformation else original - ) - fields = fields if fields else {k:k for k in original} - uuid_spec = fields.get("uuid") - name_spec = fields.get("name") - synonyms_spec = fields.get("synonyms") - context_spec = fields.get("context") - unit_spec = fields.get("unit") - cas_spec = fields.get("cas") - - self.id = generate_flow_id(original) - self.conversion_factor = transformation.get("conversion_factor") if transformation else None - self.uuid = jp.extract(uuid_spec, transformed) - self.uuid_raw_value = jp.extract(uuid_spec, original) - self.uuid_raw_object = jp.extract(uuid_spec, original, "object") - self.name = FlowProperty.from_dict(transformed, name_spec) - self.name_raw_value = jp.extract(name_spec, original) - self.name_raw_object = jp.extract(name_spec, original, "object") - self.synonyms = FlowProperty.from_dict(transformed, synonyms_spec) - self.synonyms_raw_value = jp.extract(synonyms_spec, original) - self.synonyms_raw_object = jp.extract(synonyms_spec, original, "object") - self.context = Context.from_dict(transformed, context_spec) - self.context_raw_value = jp.extract(context_spec, original) - self.context_raw_object = jp.extract(context_spec, original, "object") - self.unit = Unit.from_dict(transformed, fields["unit"]) - self.unit_raw_value = jp.extract(unit_spec, original) - self.unit_raw_object = jp.extract(unit_spec, original, "object") - self.cas = CAS(jp.extract(cas_spec, transformed)) - self.cas_raw_value = jp.extract(cas_spec, original) - self.cas_raw_object = jp.extract(cas_spec, original, "object") - self.raw = original - self.transformed = transformed - - def __repr__(self) -> str: - transformed = "*" if self.raw != self.transformed else "" - return f"{self.name_raw_value}{transformed} (in {self.unit.raw_value}) <{self.context.raw_value}> [uuid:{self.uuid_raw_value}]" - - def __eq__(self, other): - return self.id == other.id - - def __hash__(self): - return hash(self.id) - - def __lt__(self, other): - return self.name.raw_value < other.name.raw_value diff --git a/flowmapper/flowmap.py b/flowmapper/flowmap.py deleted file mode 100644 index d82bf0e..0000000 --- a/flowmapper/flowmap.py +++ /dev/null @@ -1,397 +0,0 @@ -from functools import cached_property -from .flow import Flow -from .match import match_rules, format_match_result -from .unit import Unit -from tqdm import tqdm -from typing import Callable, Optional -import pandas as pd -from pathlib import Path -import json -from collections import Counter - -class Flowmap: - """ - Crosswalk of flows from a source flow list to a target flow list. - - This class provides functionalities to map flows between different flow lists using a series of predefined match rules. - - Attributes - ---------- - source_flows : list[Flow] - The list of (unique) source flows to be mapped. - source_flows_nomatch : list[Flow] - The list of (unique) source flows that do not match any rule. - target_flows : list[Flow] - The list of target flows for mapping. - target_flows_nomatch : list[Flow] - The list of target flows that do not match any rule. - - """ - def __init__( - self, - source_flows: list[Flow], - target_flows: list[Flow], - rules: list[Callable[..., bool]] = None, - nomatch_rules: list[Callable[..., bool]] = None, - disable_progress: bool = False, - ): - """ - Initializes the Flowmap with source and target flows, along with optional matching rules. - - Duplicated flows are removed from both source and targets lists. - - Parameters - ---------- - source_flows : list[Flow] - The list of source flows to be mapped. - target_flows : list[Flow] - The list of target flows for mapping. - rules : list[Callable[..., bool]], optional - Custom rules for matching source flows to target flows. Default is the set of rules defined in `match_rules`. - nomatch_rules : list[Callable[..., bool]], optional - Rules to identify flows that should not be matched. - disable_progress : bool, optional - If True, progress bar display during the mapping process is disabled. - - """ - self.disable_progress = disable_progress - self.rules = rules if rules else match_rules() - if nomatch_rules: - self.source_flows = [] - self.source_flows_nomatch = [] - - for flow in source_flows: - matched = False - for rule in nomatch_rules: - if rule(flow): - self.source_flows_nomatch.append(flow) - matched = True - break - if not matched: - self.source_flows.append(flow) - self.source_flows = list(dict.fromkeys(self.source_flows)) - self.source_flows_nomatch = list(dict.fromkeys(self.source_flows_nomatch)) - - self.target_flows = [] - self.target_flows_nomatch = [] - - for flow in target_flows: - matched = False - for rule in nomatch_rules: - if rule(flow): - self.target_flows_nomatch.append(flow) - matched = True - break - if not matched: - self.target_flows.append(flow) - self.target_flows = list(dict.fromkeys(self.target_flows)) - self.target_flows_nomatch = list(dict.fromkeys(self.target_flows_nomatch)) - else: - self.source_flows = list(dict.fromkeys(source_flows)) - self.source_flows_nomatch = [] - self.target_flows = list(dict.fromkeys(target_flows)) - self.target_flows_nomatch = [] - - @cached_property - def mappings(self): - """ - Generates and returns a list of mappings from source flows to target flows based on the defined rules. - - Each mapping includes the source flow, target flow, conversion factor, the rule that determined the match, and additional information. - - A single match using the match rule with highest priority is returned for each source flow. - - Returns - ------- - list[dict] - A list of dictionaries containing the mapping details. - - """ - all_mappings = [] - for s in tqdm(self.source_flows, disable=self.disable_progress): - for t in self.target_flows: - for rule in self.rules: - is_match = rule(s, t) - if is_match: - all_mappings.append( - {'from': s, - 'to': t, - 'conversion_factor': s.conversion_factor if s.conversion_factor else s.unit.conversion_factor(t.unit), - 'match_rule': rule.__name__, - 'match_rule_priority': self.rules.index(rule), - 'info': is_match} - ) - break - result = [] - seen_sources = set() - sorted_mappings = sorted(all_mappings, key=lambda x: (x['from'], x['match_rule_priority'])) - for mapping in sorted_mappings: - if mapping['from'] not in seen_sources: - result.append(mapping) - seen_sources.add(mapping['from']) - - return result - - @cached_property - def _matched_source_flows_ids(self): - return {map_entry['from'].id for map_entry in self.mappings} - - @cached_property - def _matched_target_flows_ids(self): - return {map_entry['to'].id for map_entry in self.mappings} - - @cached_property - def matched_source(self): - """ - Provides a list of source flows that have been successfully matched to target flows. - - Returns - ------- - list[Flow] - A list of matched source flow objects. - - """ - result = [ - flow - for flow in self.source_flows - if flow.id in self._matched_source_flows_ids - ] - return result - - @cached_property - def unmatched_source(self): - """ - Provides a list of source flows that have not been matched to any target flows. - - Returns - ------- - list[Flow] - A list of unmatched source flow objects. - - """ - result = [ - flow - for flow in self.source_flows - if flow.id not in self._matched_source_flows_ids - ] - return result - - @cached_property - def matched_source_statistics(self): - """ - Calculates statistics for matched source flows, including the number of matches and the matching percentage for each context. - - Returns - ------- - pandas.DataFrame - A DataFrame containing matching statistics for source flows. - - """ - matched = Counter([flow.context.value for flow in self.matched_source]) - matched = pd.Series(matched).reset_index() - matched.columns = ['context', 'matched'] - - total = Counter([flow.context.value for flow in self.source_flows]) - total = pd.Series(total).reset_index() - total.columns = ['context', 'total'] - - df = pd.merge(matched, total, on='context', how='outer') - df = df.fillna(0).astype({'matched': 'int', 'total': 'int'}) - - df['percent'] = df.matched / df.total - result = df.sort_values('percent') - return result - - @cached_property - def matched_target(self): - """ - Provides a list of target flows that have been successfully matched to source flows. - - Returns - ------- - list[Flow] - A list of matched target flow objects. - - """ - result = [ - flow - for flow in self.target_flows - if flow.id in self._matched_target_flows_ids - ] - return result - - @cached_property - def unmatched_target(self): - """ - Provides a list of target flows that have not been matched to any source flows. - - Returns - ------- - list[Flow] - A list of unmatched target flow objects. - - """ - result = [ - flow - for flow in self.target_flows - if flow.id not in self._matched_target_flows_ids - ] - return result - - @cached_property - def matched_target_statistics(self): - """ - Calculates statistics for matched target flows, including the number of matches and the matching percentage for each context. - - Returns - ------- - pandas.DataFrame - A DataFrame containing matching statistics for target flows. - - """ - matched = Counter([flow.context.value for flow in self.matched_target]) - matched = pd.Series(matched).reset_index() - matched.columns = ['context', 'matched'] - - total = Counter([flow.context.value for flow in self.target_flows]) - total = pd.Series(total).reset_index() - total.columns = ['context', 'total'] - - df = pd.merge(matched, total, on='context', how='outer') - df = df.fillna(0).astype({'matched': 'int', 'total': 'int'}) - - df['percent'] = df.matched / df.total - result = df.sort_values('percent') - return result - - def statistics(self): - """ - Prints out summary statistics for the flow mapping process. - - """ - source_msg = ( - f"{len(self.source_flows)} source flows ({len(self.source_flows_nomatch)} excluded)..." - if self.source_flows_nomatch - else f"{len(self.source_flows)} source flows..." - ) - print(source_msg) - target_msg = ( - f"{len(self.target_flows)} target flows ({len(self.target_flows_nomatch)} excluded)..." - if self.target_flows_nomatch - else f"{len(self.target_flows)} target flows..." - ) - print(target_msg) - print( - f"{len(self.mappings)} mappings ({len(self.matched_source) / len(self.source_flows):.2%} of total)." - ) - cardinalities = dict(Counter([x['cardinality'] for x in self._cardinalities])) - print(f"Mappings cardinalities: {str(cardinalities)}") - - @cached_property - def _cardinalities(self): - """ - Calculates and returns the cardinalities of mappings between source and target flows. - - Returns - ------- - list[dict] - A sorted list of dictionaries, each indicating the cardinality relationship between a pair of source and target flows. - - """ - mappings = [(mapentry['from'].id, mapentry['to'].id) for mapentry in self.mappings] - lhs_counts = Counter([pair[0] for pair in mappings]) - rhs_counts = Counter([pair[1] for pair in mappings]) - - result = [] - - for lhs, rhs in mappings: - lhs_count = lhs_counts[lhs] - rhs_count = rhs_counts[rhs] - if lhs_count == 1 and rhs_count == 1: - result.append({"from": lhs, "to": rhs, "cardinality": "1:1"}) - elif lhs_count == 1 and rhs_count > 1: - result.append({"from": lhs, "to": rhs, "cardinality": "N:1"}) - elif lhs_count > 1 and rhs_count == 1: - result.append({"from": lhs, "to": rhs, "cardinality": "1:N"}) - elif lhs_count > 1 and rhs_count > 1: - result.append({"from": lhs, "to": rhs, "cardinality": "N:M"}) - - return sorted(result, key = lambda x: x['from']) - - def to_randonneur(self, path: Optional[Path] = None): - """ - Export mappings using randonneur data migration file format. - - Parameters - ---------- - path : Path, optional - If provided export the output file to disk. - - Returns - ------- - list[dict] - A list of dictionaries representing the formatted mapping results. - - """ - result = [ - format_match_result(map_entry['from'], - map_entry['to'], - map_entry['conversion_factor'], - map_entry['info']) - for map_entry in self.mappings - ] - - if not path: - return result - else: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, 'w') as fs: - json.dump(result, fs, indent=2) - - - def to_glad(self, path: Optional[Path] = None, ensure_id: bool = False): - """ - Export mappings using GLAD flow mapping format, optionally ensuring each flow has an identifier. - - Formats the mapping results according to Global LCA Data Access (GLAD) network initiative flow mapping format. - - Parameters - ---------- - path : Path, optional - If provided export the output file to disk. - ensure_id : bool, optional - If True, ensures each flow has an identifier, default is False. - - Returns - ------- - pandas.DataFrame - A DataFrame containing the formatted mapping results in GLAD format. - - """ - data = [] - for map_entry in self.mappings: - source_flow_id = map_entry['from'].uuid_raw_value if map_entry['from'].uuid_raw_value or not ensure_id else map_entry['from'].id - row = { - 'SourceFlowName': map_entry['from'].name_raw_value, - 'SourceFlowUUID': source_flow_id, - 'SourceFlowContext': '/'.join(map_entry['from'].context_raw_value), - 'SourceUnit': map_entry['from'].unit_raw_value, - 'MatchCondition': '', - 'ConversionFactor': map_entry['conversion_factor'], - 'TargetFlowName': map_entry['to'].name_raw_value, - 'TargetFlowUUID': map_entry['to'].uuid_raw_value, - 'TargetFlowContext': map_entry['to'].context.raw_value, - 'TargetUnit': map_entry['to'].unit.raw_value, - 'MemoMapper': map_entry['info'].get('comment') - } - data.append(row) - - result = pd.DataFrame(data) - - if not path: - return result - else: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - result.to_excel(path, index = False) diff --git a/flowmapper/flowproperty.py b/flowmapper/flowproperty.py deleted file mode 100644 index 1cb94df..0000000 --- a/flowmapper/flowproperty.py +++ /dev/null @@ -1,60 +0,0 @@ -from dataclasses import asdict, dataclass, field -import flowmapper.jsonpath as jp -from .utils import normalize_str - -@dataclass -class FlowProperty: - value: str = None - raw_value: str = None - raw_object: dict = field(default_factory=lambda: {}) - - def __post_init__(self): - raw_value = self.value - self.value = self.normalize(self.value) - if raw_value != self.value: - self.raw_value = raw_value - - @classmethod - def from_dict(cls, d, spec): - if spec: - key = jp.root(spec) - value = jp.extract(spec, d) - result = FlowProperty( - value = cls.normalize(value), - raw_value = value, - raw_object = {key: d.get(key)}, - ) - else: - result = FlowProperty(None) - return result - - def to_dict(self): - return asdict(self) - - def __eq__(self, other): - result = False - if self and other: - if isinstance(other, FlowProperty): - result = self.value == other.value - else: - result = self.value == other - return result - - def __bool__(self): - return bool(self.value) - - def __repr__(self): - return str(self.value) - - def __hash__(self): - return hash(self.value) - - @staticmethod - def normalize(x): - if x is None: - result = None - elif isinstance(x, str): - result = normalize_str(x) - else: - result = [normalize_str(s) for s in x if s] - return result diff --git a/flowmapper/jsonpath.py b/flowmapper/jsonpath.py deleted file mode 100644 index 581bff5..0000000 --- a/flowmapper/jsonpath.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Literal -from glom import Coalesce, glom - -def root(spec): - result = None - if isinstance(spec, str): - result = spec.split('.')[0] - else: - try: - result = next(iter(spec)) - except TypeError: - pass - return result - -def extract(spec, data, type: Literal['value', 'object'] = 'value'): - if spec and type == 'value': - return glom(data, Coalesce(spec, default = '')) - if spec and type == 'object' and data.get(root(spec)): - key = root(spec) - return {key: data[key]} - else: - return None diff --git a/flowmapper/match.py b/flowmapper/match.py deleted file mode 100644 index ba6c45c..0000000 --- a/flowmapper/match.py +++ /dev/null @@ -1,108 +0,0 @@ -import logging -from .flow import Flow -from .utils import rm_parentheses_roman_numerals, extract_country_code, rm_roman_numerals_ionic_state - -logger = logging.getLogger(__name__) - -def format_match_result(s: Flow, t: Flow, conversion_factor: float, match_info: dict): - source_result = { - **s.name_raw_object, - **s.context_raw_object, - **s.unit_raw_object - } - if s.uuid: - source_result.update(**s.uuid_raw_object) - - target_result = { - 'uuid': t.uuid, - 'name': t.name_raw_value, - 'context': t.context.raw_value, - 'unit': t.unit.raw_value - } - if match_info.get('location'): - target_result.update({'location': match_info['location']}) - - result = { - 'source': source_result, - 'target': target_result, - 'conversion_factor': conversion_factor, - 'comment': match_info['comment'] - } - return result - -def match_identical_uuid(s: Flow, t: Flow, comment: str = 'Identical uuid'): - is_match = True if s.uuid and t.uuid and s.uuid == t.uuid else False - if is_match: - return {'comment': comment} - -def match_identical_names_in_synonyms(s: Flow, t: Flow, comment: str = 'Identical synonyms'): - is_match = True if t.synonyms and s.name.value in t.synonyms.value and s.context == t.context else False - if is_match: - return {'comment': comment} - -def match_identical_cas_numbers(s: Flow, t: Flow, comment: str = 'Identical CAS numbers'): - is_match = s.cas == t.cas and s.context == t.context - if is_match: - return {'comment': comment} - -def match_identical_names(s: Flow, t: Flow, comment = 'Identical names'): - is_match = s.name == t.name and s.context == t.context - - if is_match: - return {'comment': comment} - -def match_identical_names_except_missing_suffix(s: Flow, t: Flow, suffix, comment = 'Identical names except missing suffix'): - is_match = ( - (f"{s.name}, {suffix}" == t.name) or - (f"{t.name}, {suffix}" == s.name) or - (f"{s.name} {suffix}" == t.name) or - (f"{t.name} {suffix}" == s.name) - ) and s.context == t.context - - if is_match: - return {'comment': comment} - -def match_names_with_roman_numerals_in_parentheses(s: Flow, t: Flow, comment = 'With/without roman numerals in parentheses'): - is_match = rm_parentheses_roman_numerals(s.name.value) == rm_parentheses_roman_numerals(t.name.value) and s.context == t.context - - if is_match: - return {'comment': comment} - -def match_names_with_country_codes(s: Flow, t: Flow, comment = 'Names with country code'): - s_name, s_location = extract_country_code(s.name.value) - is_match = s_location and s_name == t.name and s.context == t.context - - if is_match: - return {'comment': comment, 'location': s_location.upper()} - -def match_non_ionic_state(s: Flow, t: Flow, comment = 'Non-ionic state if no better match'): - is_match = rm_roman_numerals_ionic_state(s.name.value) == t.name and s.context == t.context - - if is_match: - return {'comment': comment} - -def match_biogenic_to_non_fossil(s: Flow, t: Flow, comment = 'Biogenic to non-fossil if no better match'): - is_match = s.name.value.removesuffix(', biogenic') == t.name.value.removesuffix(', non-fossil') and s.context == t.context - - if is_match: - return {'comment': comment} - -def match_resources_with_suffix_in_ground(s: Flow, t: Flow): - return match_identical_names_except_missing_suffix(s, t, suffix = 'in ground', comment = 'Resources with suffix in ground') - -def match_emissions_with_suffix_ion(s: Flow, t: Flow): - return match_identical_names_except_missing_suffix(s, t, suffix = 'ion', comment = 'Match emissions with suffix ion') - -def match_rules(): - return [ - match_identical_uuid, - match_identical_names, - match_resources_with_suffix_in_ground, - match_emissions_with_suffix_ion, - match_names_with_roman_numerals_in_parentheses, - match_names_with_country_codes, - match_identical_cas_numbers, - match_non_ionic_state, - match_biogenic_to_non_fossil, - match_identical_names_in_synonyms, - ] diff --git a/flowmapper/unit.py b/flowmapper/unit.py deleted file mode 100644 index 776cfb5..0000000 --- a/flowmapper/unit.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations -from dataclasses import asdict, dataclass, field -from typing import Any -import flowmapper.jsonpath as jp -from pint import UnitRegistry, errors -import importlib.resources as resource -from .utils import normalize_str -from .constants import UNITS_NORMALIZATION - -ureg = UnitRegistry() - -with resource.as_file(resource.files('flowmapper').joinpath('data/units.txt')) as filepath: - ureg.load_definitions(filepath) - -@dataclass -class Unit: - value: str - raw_value: str = "" - raw_object: dict = field(default_factory=lambda: {}) - - def __post_init__(self): - if not self.raw_value: - self.raw_value = self.value - self.value = self.normalize(self.raw_value) - - @classmethod - def from_dict(cls, d, spec): - key = jp.root(spec) - value = jp.extract(spec, d) - result = Unit( - value = None, - raw_value = value, - raw_object = {key: d.get(key)}, - ) - return result - - def to_dict(self): - return asdict(self) - - def __eq__(self, other): - return self.value == other.value - - def conversion_factor(self, to: Unit): - if self.value == to.value: - result = 1.0 - else: - try: - result = ureg(self.value).to(ureg(to.value)).magnitude - except errors.DimensionalityError: - result = float('nan') - return result - - @staticmethod - def normalize(x): - s = normalize_str(x) - return UNITS_NORMALIZATION.get(s, s) diff --git a/flowmapper/utils.py b/flowmapper/utils.py deleted file mode 100644 index 9daddb4..0000000 --- a/flowmapper/utils.py +++ /dev/null @@ -1,122 +0,0 @@ -import copy -import json -from collections import Counter -from pathlib import Path -import hashlib -import re -from typing import Optional, Union -import unicodedata -from randonneur import migrate_datasets -try: - import tomllib -except ModuleNotFoundError: - import tomli as tomllib -import importlib.util - -def import_module(filepath): - filepath = Path(filepath) - module_name = filepath.stem - spec = importlib.util.spec_from_file_location(module_name, filepath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - -def read_field_mapping(filepath: Path): - module = import_module(filepath) - fields = getattr(module, 'config', None) - if not fields: - raise Exception(f'{filepath} does not define a dict named config with field mapping information.') - - result = { - 'source': {}, - 'target': {} - } - - for key, values in fields.items(): - if len(values) == 1: - result['source'][key] = values[0] - result['target'][key] = values[0] - else: - result['source'][key] = values[0] - result['target'][key] = values[1] - - return result - - -def generate_flow_id(flow: dict): - flow_str = json.dumps(flow, sort_keys=True) - result = hashlib.md5(flow_str.encode('utf-8')).hexdigest() - return result - - -def read_flowlist(filepath: Path): - with open(filepath, 'r') as fs: - result = json.load(fs) - return result - -def read_migration_files(*filepaths: Union[str, Path]): - """ - Read and aggregate migration data from multiple JSON files. - - This function opens and reads a series of JSON files, each containing migration data as a list of dicts without the change type. - It aggregates all changes into a single list and returns it wrapped in a dictionary - under the change type 'update'. - - Parameters - ---------- - *filepaths : Path - Variable length argument list of Path objects. - - Returns - ------- - dict - A dictionary containing a single key 'update', which maps to a list. This list is - an aggregation of the data from all the JSON files read. - """ - migration_data = [] - - for filepath in filepaths: - filepath = Path(filepath) - with open(filepath, 'r') as fs: - migration_data.extend(json.load(fs)) - - result = {'update': migration_data} - return result - -def rm_parentheses_roman_numerals(s: str): - pattern = r'\(\s*([ivxlcdm]+)\s*\)' - return re.sub(pattern, r'\1', s) - -def rm_roman_numerals_ionic_state(s: str): - pattern = r'\s*\(\s*[ivxlcdm]+\s*\)' - return re.sub(pattern, '', s) - -def extract_country_code(s: str) -> tuple[str, Optional[str]]: - # Regex to find a two-letter uppercase code following a comma and optional whitespace - match = re.search(r',\s*([a-z]{2})$', s) - - if match: - # Extract the country code and the preceding part of the string - country_code = match.group(1) - rest_of_string = s[:match.start()].strip() - return (rest_of_string, country_code) - else: - return (s, None) - -def normalize_str(s): - return unicodedata.normalize('NFC', s).strip().lower() - -def transform_flow(flow, transformation): - result = copy.copy(flow) - result.update(transformation['target']) - return result - -def matcher(source, target): - return all(target.get(key) == value for key, value in source.items()) - -def find_transformation(flow, transformations): - if not transformations: - return None - for transformation in transformations['update']: - if matcher(transformation['source'], flow): - return transformation diff --git a/pyproject.toml b/pyproject.toml index 8d8cd9f..7eb164e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,11 @@ build-backend = "setuptools.build_meta" [project] name = "flowmapper" authors = [ - { name="Francisco Alves de Oliveira Júnior", email="fjunior.alves.oliveira@gmail.com" } + { name="Francisco Alves de Oliveira Júnior", email="fjunior.alves.oliveira@gmail.com" }, + { name="Chris Mute", email="cmutel@gmail.com" } ] maintainers = [ - { name="Francisco Alves de Oliveira Júnior", email="fjunior.alves.oliveira@gmail.com" } + { name="Chris Mute", email="cmutel@gmail.com" } ] description = "Generate mappings between elementary flow lists" readme = "README.md" @@ -19,28 +20,35 @@ classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Natural Language :: English", "Operating System :: OS Independent", "Topic :: Scientific/Engineering" ] -requires-python = ">=3.9" +requires-python = ">=3.11" dependencies = [ - "glom", - "tqdm", + "bw_simapro_csv", "pandas[excel]", - "typer", "pint", - "randonneur" + "pydantic", + "pyecospold", + "randonneur>=0.6", + "randonneur_data", + "RapidFuzz", + "roman", + "structlog", + "tqdm", + "typer", + "xmltodict", ] [project.urls] -source = "https://github.com/fjuniorr/flowmapper" -homepage = "https://github.com/fjuniorr/flowmapper" -tracker = "https://github.com/fjuniorr/flowmapper/issues" +source = "https://github.com/cmutel/flowmapper" +homepage = "https://github.com/cmutel/flowmapper" +tracker = "https://github.com/cmutel/flowmapper/issues" [project.optional-dependencies] # Getting recursive dependencies to work is a pain, this @@ -51,11 +59,11 @@ testing = [ "pytest-cov", "python-coveralls", "deepdiff", - "syrupy", ] dev = [ "build", "pre-commit", + "pyinstrument", "pylint", "pytest", "pytest-cov", @@ -68,13 +76,15 @@ flowmapper = "flowmapper.cli:app" [tool.setuptools] license-files = ["LICENSE"] include-package-data = true -packages = ["flowmapper"] + +[tool.setuptools.packages.find] +where = ["src"] [tool.setuptools.dynamic] version = {attr = "flowmapper.__version__"} [tool.setuptools.package-data] -flowmapper = ["data/units.txt"] +flowmapper = ["data/*.txt", "data/*.json", "manual_matching/data/**/*.json", "manual_matching/results/**/*.json"] [tool.pytest.ini_options] addopts = "--cov flowmapper --cov-report term-missing --verbose" @@ -109,3 +119,6 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true ensure_newline_before_comments = true + +[tool.ruff] +line-length = 100 diff --git a/src/flowmapper/__init__.py b/src/flowmapper/__init__.py new file mode 100644 index 0000000..b31b3c0 --- /dev/null +++ b/src/flowmapper/__init__.py @@ -0,0 +1,21 @@ +__all__ = ( + "__version__", + "CASField", + "ContextField", + "Flow", + "Flowmap", + "flowmapper", + "Match", + "MatchCondition", + "NormalizedFlow", + "UnitField", +) + +__version__ = "0.4.2" + +from flowmapper.cas import CASField +from flowmapper.context import ContextField +from flowmapper.domain import Flow, Match, MatchCondition, NormalizedFlow +from flowmapper.flowmap import Flowmap +from flowmapper.main import flowmapper +from flowmapper.unit import UnitField diff --git a/src/flowmapper/cas.py b/src/flowmapper/cas.py new file mode 100644 index 0000000..880bb3d --- /dev/null +++ b/src/flowmapper/cas.py @@ -0,0 +1,69 @@ +import re +from collections import UserString +from functools import cached_property +from typing import Any + +valid_cas = re.compile(r"^\s*[0-9]{2,7}-[0-9]{2}-[0-9]{1}\s*$") + + +class CASField(UserString): + def __init__(self, string: str): + if not isinstance(string, (str, UserString)): + raise TypeError( + f"CASField takes only `str`, but got {type(string)} for {string}" + ) + if not valid_cas.search(str(string)): + raise ValueError(f"Given input is not valid CAS formatting: '{string}'") + super().__init__(str(string)) + + def __eq__(self, other: Any) -> bool: + if isinstance(other, CASField): + return self.data == other.data + elif isinstance(other, (str, UserString)): + other_cas = CASField.from_string(str(other)) + if other_cas is None: + return False + return self.data == other_cas.data + return False + + @staticmethod + def from_string(string: str | None) -> "CASField | None": + """Returns `None` if CAS number is invalid""" + if string is None or not isinstance(string, (str, UserString)): + return None + new_cas = CASField(string.strip().lstrip("0").strip()) + if not new_cas.valid(): + return None + return new_cas + + @property + def digits(self) -> list[int]: + return [int(d) for d in self.data.replace("-", "")] + + def export(self): + return "{}-{}-{}".format( + "".join([str(x) for x in self.digits[:-3]]), + "".join([str(x) for x in self.digits[-3:-1]]), + self.digits[-1], + ) + + @cached_property + def check_digit_expected(self): + """ + Expected digit acording to https://www.cas.org/support/documentation/chemical-substances/checkdig algorithm + """ + result = ( + sum( + [ + index * value + for index, value in enumerate(self.digits[-2::-1], start=1) + ] + ) + % 10 + ) + return result + + def valid(self): + return (self.digits[-1] == self.check_digit_expected) and bool( + valid_cas.search(self.data) + ) diff --git a/src/flowmapper/cli.py b/src/flowmapper/cli.py new file mode 100644 index 0000000..9775170 --- /dev/null +++ b/src/flowmapper/cli.py @@ -0,0 +1,153 @@ +import importlib.metadata +from pathlib import Path +from typing import Annotated + +import structlog +import typer + +from flowmapper.extraction import ( + ecospold2_biosphere_extractor, + simapro_csv_biosphere_extractor, +) +from flowmapper.main import flowmapper + +try: + from pyinstrument import Profiler +except ImportError: + Profiler = None + + +logger = structlog.get_logger("flowmapper") + +app = typer.Typer() + + +def version_callback(value: bool): + if value: + print(f"flowmapper, version {importlib.metadata.version('flowmapper')}") + raise typer.Exit() + + +@app.callback() +def main( + version: Annotated[ + bool | None, + typer.Option("--version", callback=version_callback, is_eager=True), + ] = None, +): + """ + Generate mappings between elementary flows lists + """ + + +@app.command() +def map( + source: Annotated[Path, typer.Argument(help="Path to source flow list")], + target: Annotated[Path, typer.Argument(help="Path to target flow list")], + output_dir: Annotated[ + Path, typer.Option(help="Directory to save mapping and diagnostics files") + ] = Path("."), + default_transformations: Annotated[ + bool, typer.Option(help="Include default context and unit transformations?") + ] = True, + transformations: Annotated[ + list[Path] | None, + typer.Option( + "--transformations", + "-t", + help="Randonneur data migration file with changes to be applied to source flows before matching. Can be included multiple times.", + ), + ] = None, + unmatched_source: Annotated[ + bool, + typer.Option(help="Write original source unmatched flows into separate file?"), + ] = True, + unmatched_target: Annotated[ + bool, + typer.Option(help="Write original target unmatched flows into separate file?"), + ] = True, + matched_source: Annotated[ + bool, + typer.Option(help="Write original source matched flows into separate file?"), + ] = False, + matched_target: Annotated[ + bool, + typer.Option(help="Write original target matched flows into separate file?"), + ] = False, + profile: Annotated[ + bool, + typer.Option(help="Profile matching code with pyinstrument"), + ] = False, +): + # Default generic mapping for JSON flow lists + generic_mapping = { + "expression language": "JSONPath", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + } + + if profile: + if Profiler is None: + raise ImportError("`pyinstrument` not installed") + profiler = Profiler(interval=0.01) + profiler.start() + + result = flowmapper( + source=source, + target=target, + mapping_source=generic_mapping, + mapping_target=generic_mapping, + source_id=source.stem, + target_id=target.stem, + contributors=[ + { + "title": "flowmapper", + "roles": ["author"], + "path": "https://github.com/cmutel/flowmapper", + } + ], + output_dir=output_dir, + default_transformations=default_transformations, + transformations=transformations, + unmatched_source=unmatched_source, + unmatched_target=unmatched_target, + matched_source=matched_source, + matched_target=matched_target, + ) + + if profile: + profiler.stop() + with open(f"{source.stem}-{target.stem}.html", "w") as f: + f.write(profiler.output_html()) + + return result + + +@app.command() +def extract_simapro_csv( + simapro_csv_filepath: Annotated[ + Path, typer.Argument(help="Path to SimaPro CSV input file") + ], + output_filepath: Annotated[ + Path, typer.Argument(help="File path for JSON results data") + ], +) -> None: + simapro_csv_biosphere_extractor(simapro_csv_filepath, output_filepath) + + +@app.command() +def extract_ecospold2( + elementary_exchanges_filepath: Annotated[ + Path, typer.Argument(help="Path to source `ElementaryExchanges.xml` file") + ], + output_filepath: Annotated[ + Path, typer.Argument(help="File path for JSON results data") + ], +) -> None: + ecospold2_biosphere_extractor(elementary_exchanges_filepath, output_filepath) diff --git a/src/flowmapper/constants.py b/src/flowmapper/constants.py new file mode 100644 index 0000000..591c5c8 --- /dev/null +++ b/src/flowmapper/constants.py @@ -0,0 +1,11 @@ +RESOURCE_PARENT_CATEGORY = { + "natural resources", + "natural resource", + "resources", + "resource", + "land use", + "economic", + "social", + "raw materials", + "raw", +} diff --git a/src/flowmapper/context.py b/src/flowmapper/context.py new file mode 100644 index 0000000..b610296 --- /dev/null +++ b/src/flowmapper/context.py @@ -0,0 +1,75 @@ +from collections.abc import Iterable +from typing import Any, Self + +from flowmapper.utils import as_normalized_tuple + +RESOURCE_CATEGORY = { + "natural resources", + "natural resource", + "resources", + "resource", + "land use", + "economic", + "social", + "raw materials", + "raw", +} + + +class ContextField: + def __init__(self, value: str | list[str] | tuple[str]): + self.value = value + + def normalize(self, obj: Any | None = None, mapping: dict | None = None) -> Self: + return type(self)(value=as_normalized_tuple(value=obj or self.value)) + + def is_resource(self) -> bool: + if isinstance(self.value, str): + return any(cat in self.value.lower() for cat in RESOURCE_CATEGORY) + else: + lowered = [elem.lower() for elem in self.value] + return any(cat in lowered for cat in RESOURCE_CATEGORY) + + def as_tuple(self) -> tuple | str: + if isinstance(self.value, str): + return self.value + return tuple(self.value) + + def export_as_string(self, join_character: str = "✂️"): + if isinstance(self.value, (list, tuple)): + return join_character.join(self.value) + return self.value + + def __iter__(self) -> Iterable: + return iter(self.value) + + def __eq__(self, other: Any) -> bool: + if self and other and isinstance(other, ContextField): + return self.value == other.value + else: + try: + return self.value == self.normalize(other).value + except ValueError: + return False + + def __repr__(self) -> str: + return str(self.value) + + def __bool__(self) -> bool: + return bool(self.value) + + def __hash__(self) -> int: + return hash(self.value) + + def __contains__(self, other: Any) -> bool: + """`self` context is more generic than the `other` context. + + ```python + Context("a/b") in Context("a/b/c") + >>> True + ``` + + """ + if not isinstance(other, ContextField): + return False + return self.value == other.value[: len(self.value)] diff --git a/src/flowmapper/data/manual_name_match_simapro_ecoinvent_3.8.json b/src/flowmapper/data/manual_name_match_simapro_ecoinvent_3.8.json new file mode 100644 index 0000000..c3e0b63 --- /dev/null +++ b/src/flowmapper/data/manual_name_match_simapro_ecoinvent_3.8.json @@ -0,0 +1,2322 @@ +[ + [ + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, freezing {GLO}| market for transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refri(...)_16 | Cut-off", + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, freezing {GLO}| transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, freezing | Cut-off" + ], + [ + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, cooling {GLO}| market for transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrig(...)_8 | Cut-off", + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, cooling {GLO}| transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, cooling | Cut-off" + ], + [ + "Occupation, agriculture", + "Occupation, annual crop" + ], + [ + "Occupation, dump site, benthos", + "Occupation, seabed, drilling and mining" + ], + [ + "Occupation, forest, intensive, normal", + "Occupation, forest, intensive" + ], + [ + "Occupation, forest, intensive, short-cycle", + "Occupation, forest, intensive" + ], + [ + "Occupation, forest, used", + "Occupation, forest, intensive" + ], + [ + "Occupation, grassland/pasture/meadow", + "Occupation, pasture, man made" + ], + [ + "Occupation, industrial area, benthos", + "Occupation, industrial area" + ], + [ + "Occupation, industrial area, built up", + "Occupation, industrial area" + ], + [ + "Occupation, industrial area, vegetation", + "Occupation, industrial area" + ], + [ + "Occupation, permanent crop, fruit, intensive", + "Occupation, permanent crop, irrigated, intensive" + ], + [ + "Occupation, sea and ocean", + "Occupation, seabed, unspecified" + ], + [ + "Occupation, traffic area", + "Occupation, traffic area, road network" + ], + [ + "Occupation, traffic area, road embankment", + "Occupation, traffic area, rail/road embankment" + ], + [ + "Occupation, unspecified, used", + "Occupation, unspecified" + ], + [ + "Occupation, water bodies, artificial", + "Occupation, lake, artificial" + ], + [ + "Occupation, lakes, artificial", + "Occupation, lake, artificial" + ], + [ + "Occupation, rivers, artificial", + "Occupation, river, artificial" + ], + [ + "Transformation, to lakes, artificial", + "Transformation, to lake, artificial" + ], + [ + "Transformation, to rivers, artificial", + "Transformation, to river, artificial" + ], + [ + "Occupation, water courses, artificial", + "Occupation, river, artificial" + ], + [ + "Occupation, wetland", + "Occupation, inland waterbody, unspecified" + ], + [ + "Transformation, from agriculture", + "Transformation, from annual crop" + ], + [ + "Transformation, from annual crop, non-irrigated, fallow", + "Transformation, from pasture, man made" + ], + [ + "Transformation, from forest, intensive, clear-cutting", + "Transformation, from forest, intensive" + ], + [ + "Transformation, from forest, used", + "Transformation, from forest, intensive" + ], + [ + "Transformation, from grassland", + "Transformation, from grassland, natural (non-use)" + ], + [ + "Transformation, from grassland/pasture/meadow", + "Transformation, from pasture, man made" + ], + [ + "Transformation, from industrial area, benthos", + "Transformation, from industrial area" + ], + [ + "Transformation, from industrial area, built up", + "Transformation, from industrial area" + ], + [ + "Transformation, from industrial area, vegetation", + "Transformation, from industrial area" + ], + [ + "Transformation, from permanent crop, fruit", + "Transformation, from permanent crop, irrigated" + ], + [ + "Transformation, from tropical rain forest", + "Transformation, from forest, primary (non-use)" + ], + [ + "Transformation, from unspecified, used", + "Transformation, from unspecified" + ], + [ + "Transformation, to agriculture", + "Transformation, to annual crop" + ], + [ + "Transformation, to annual crop, fallow", + "Transformation, to arable land, unspecified use" + ], + [ + "Transformation, to annual crop, non-irrigated, fallow", + "Transformation, to annual crop, non-irrigated, extensive" + ], + [ + "Transformation, to dump site, benthos", + "Transformation, to seabed, drilling and mining" + ], + [ + "Transformation, to forest, intensive, clear-cutting", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to forest, intensive, normal", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to forest, intensive, short-cycle", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to forest, used", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to grassland/pasture/meadow", + "Transformation, to pasture, man made" + ], + [ + "Transformation, to industrial area, benthos", + "Transformation, to industrial area" + ], + [ + "Transformation, to industrial area, built up", + "Transformation, to industrial area" + ], + [ + "Transformation, to industrial area, vegetation", + "Transformation, to industrial area" + ], + [ + "Transformation, to permanent crop, fruit, intensive", + "Transformation, to permanent crop, irrigated, intensive" + ], + [ + "Transformation, to sea and ocean", + "Transformation, to seabed, unspecified" + ], + [ + "Transformation, to traffic area, road embankment", + "Transformation, to traffic area, rail/road embankment" + ], + [ + "Transformation, to unspecified, used", + "Transformation, to unspecified" + ], + [ + "Transformation, to water bodies, artificial", + "Transformation, to lake, artificial" + ], + [ + "Transformation, to water courses, artificial", + "Transformation, to river, artificial" + ], + [ + "Particulates, < 10 um", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, < 10 um (mobile)", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, < 10 um (stationary)", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, > 10 um (process)", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, SPM", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, diesel soot", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, unspecified", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Water/m3", + "Water" + ], + [ + "Sulfur monoxide", + "Sulfur oxides" + ], + [ + "Nitrogen, atmospheric", + "Nitrogen" + ], + [ + "Phenols, unspecified", + "Phenol" + ], + [ + "AOX (Adsorbable Organic Halogens)", + "AOX, Adsorbable Organic Halogen as Cl" + ], + [ + "AOX, Adsorbable Organic Halogen", + "AOX, Adsorbable Organic Halogen as Cl" + ], + [ + "Alkane (unspecified)", + "Hydrocarbons, aliphatic, alkanes, unspecified" + ], + [ + "Antimony, ion", + "Antimony" + ], + [ + "Formetanate", + "Formetanate hydrochloride" + ], + [ + "BOD5 (Biological Oxygen Demand)", + "BOD5, Biological Oxygen Demand" + ], + [ + "COD (Chemical Oxygen Demand)", + "COD, Chemical Oxygen Demand" + ], + [ + "Haloxyfop", + "haloxyfop-ethoxyethyl" + ], + [ + "Bauxite", + "Gangue, bauxite, in ground" + ], + [ + "Copper ore", + "Copper, Cu 0.2%, in mixed ore, in ground" + ], + [ + "Copper, Cu 0.38%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Pb 0.014%, in ore", + "Copper, Cu 0.38%, in mixed ore, in ground" + ], + [ + "Coal, 18 MJ per kg", + "Coal, hard, unspecified, in ground" + ], + [ + "Coal, brown, 10 MJ per kg", + "Coal, brown, in ground" + ], + [ + "Coal, brown, 8 MJ per kg", + "Coal, brown, in ground" + ], + [ + "Coal, hard", + "Coal, hard, unspecified, in ground" + ], + [ + "Crude oil", + "Oil, crude, in ground" + ], + [ + "Energy, from biomass", + "Energy, gross calorific value, in biomass" + ], + [ + "Energy, from coal", + "Energy, gross calorific value, in biomass" + ], + [ + "Gas, mine, off-gas, process, coal mining/kg", + "Gas, mine, off-gas, process, coal mining" + ], + [ + "Arsenic (V)", + "Arsenic" + ], + [ + "Barium (II)", + "Barium" + ], + [ + "Beryllium (II)", + "Beryllium" + ], + [ + "Cadmium (II)", + "Cadmium" + ], + [ + "Calcium", + "Calcium, ion" + ], + [ + "Chromium (III)", + "Chromium III" + ], + [ + "Chromium (VI)", + "Chromium VI" + ], + [ + "Chromium, ion", + "Chromium" + ], + [ + "Copper, ion", + "Copper" + ], + [ + "Lead (II)", + "Lead" + ], + [ + "Mercury (II)", + "Mercury" + ], + [ + "Manganese (II)", + "Manganese" + ], + [ + "Molybdenum (VI)", + "Molybdenum" + ], + [ + "Nickel (II)", + "Nickel" + ], + [ + "Oils, biogenic", + "Oils, non-fossil" + ], + [ + "Plutonium", + "Plutonium-238" + ], + [ + "Potassium (I)", + "Potassium" + ], + [ + "Ruthenium", + "Ruthenium-103" + ], + [ + "Silver (I)", + "Silver" + ], + [ + "Thallium (I)", + "Thallium" + ], + [ + "Zinc (II)", + "Zinc" + ], + [ + "Aluminum", + "Aluminium" + ], + [ + "Carbon dioxide, biogenic", + "Carbon dioxide, non-fossil" + ], + [ + "Carbon dioxide, land transformation", + "Carbon dioxide, from soil or biomass stock" + ], + [ + "Carbon monoxide, biogenic", + "Carbon monoxide, non-fossil" + ], + [ + "Methane, biogenic", + "Methane, non-fossil" + ], + [ + "Sodium tetrahydroborate", + "Sodium tetrahydridoborate" + ], + [ + "Sulphur trioxide", + "Sulfur trioxide" + ], + [ + "Bromine, 0.0023% in water", + "Bromine, 0.23% in water" + ], + [ + "Carbon, in organic matter, in soil", + "Carbon, organic, in soil or biomass stock" + ], + [ + "Copper, Cu 0.38%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Pb 0.014%, in ore, in ground", + "Copper, Cu 0.38%, in mixed ore, in ground" + ], + [ + "Gold, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore, in ground", + "Gold, Au 9.7E-4%, in mixed ore, in ground" + ], + [ + "Lead, Pb 0.014%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, in ore, in ground", + "Lead, Pb 0.014%, in mixed ore, in ground" + ], + [ + "Occupation, arable", + "Occupation, arable land, unspecified use" + ], + [ + "Occupation, arable, non-irrigated", + "Occupation, annual crop, non-irrigated" + ], + [ + "Occupation, arable, non-irrigated, diverse-intensive", + "Occupation, annual crop, non-irrigated, extensive" + ], + [ + "Occupation, arable, non-irrigated, fallow", + "Occupation, cropland fallow (non-use)" + ], + [ + "Occupation, arable, non-irrigated, monotone-intensive", + "Occupation, annual crop, non-irrigated, intensive" + ], + [ + "Occupation, forest", + "Occupation, forest, unspecified" + ], + [ + "Occupation, forest, intensive, clear-cutting", + "Occupation, forest, intensive" + ], + [ + "Occupation, pasture and meadow", + "Occupation, pasture, man made" + ], + [ + "Occupation, pasture and meadow, extensive", + "Occupation, pasture, man made, extensive" + ], + [ + "Occupation, pasture and meadow, intensive", + "Occupation, pasture, man made, intensive" + ], + [ + "Occupation, permanent crop, fruit", + "Occupation, permanent crop, irrigated" + ], + [ + "Occupation, permanent crop, fruit, extensive", + "Occupation, permanent crop, irrigated, extensive" + ], + [ + "Occupation, permanent crop, vine", + "Occupation, permanent crop, non-irrigated" + ], + [ + "Occupation, permanent crop, vine, extensive", + "Occupation, permanent crop, non-irrigated, extensive" + ], + [ + "Occupation, permanent crop, vine, intensive", + "Occupation, permanent crop, non-irrigated, intensive" + ], + [ + "Occupation, traffic area, rail embankment", + "Occupation, traffic area, rail/road embankment" + ], + [ + "Occupation, tropical rain forest", + "Occupation, forest, primary (non-use)" + ], + [ + "Occupation, unknown", + "Occupation, unspecified" + ], + [ + "Silver, Ag 9.7E-4%, Au 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore, in ground", + "Silver, Ag 9.7E-4%, in mixed ore, in ground" + ], + [ + "Transformation, from arable", + "Transformation, from arable land, unspecified use" + ], + [ + "Transformation, from arable, non-irrigated", + "Transformation, from annual crop, non-irrigated" + ], + [ + "Transformation, from arable, non-irrigated, diverse-intensive", + "Transformation, from annual crop, non-irrigated, extensive" + ], + [ + "Transformation, from arable, non-irrigated, fallow", + "Transformation, from cropland fallow (non-use)" + ], + [ + "Transformation, from arable, non-irrigated, monotone-intensive", + "Transformation, from annual crop, non-irrigated, intensive" + ], + [ + "Transformation, from dump site, benthos", + "Transformation, from seabed, drilling and mining" + ], + [ + "Transformation, from forest", + "Transformation, from forest, unspecified" + ], + [ + "Transformation, from forest, intensive, normal", + "Transformation, from forest, intensive" + ], + [ + "Transformation, from forest, intensive, short-cycle", + "Transformation, from forest, intensive" + ], + [ + "Transformation, from pasture and meadow", + "Transformation, from pasture, man made" + ], + [ + "Transformation, from pasture and meadow, extensive", + "Transformation, from pasture, man made, extensive" + ], + [ + "Transformation, from pasture and meadow, intensive", + "Transformation, from pasture, man made, intensive" + ], + [ + "Transformation, from permanent crop, fruit, extensive", + "Transformation, from permanent crop, irrigated, extensive" + ], + [ + "Transformation, from permanent crop, fruit, intensive", + "Transformation, from permanent crop, irrigated, intensive" + ], + [ + "Transformation, from permanent crop, vine", + "Transformation, from permanent crop, non-irrigated" + ], + [ + "Transformation, from permanent crop, vine, extensive", + "Transformation, from permanent crop, non-irrigated, extensive" + ], + [ + "Transformation, from permanent crop, vine, intensive", + "Transformation, from permanent crop, non-irrigated, intensive" + ], + [ + "Transformation, from sea and ocean", + "Transformation, from seabed, unspecified" + ], + [ + "Transformation, from traffic area, rail embankment", + "Transformation, from traffic area, rail/road embankment" + ], + [ + "Transformation, from traffic area, road embankment", + "Transformation, from traffic area, rail/road embankment" + ], + [ + "Transformation, from unknown", + "Transformation, from unspecified" + ], + [ + "Transformation, from water bodies, artificial", + "Transformation, from lake, artificial" + ], + [ + "Transformation, from water courses, artificial", + "Transformation, from river, artificial" + ], + [ + "Transformation, to arable", + "Transformation, to arable land, unspecified use" + ], + [ + "Transformation, to arable, non-irrigated", + "Transformation, to annual crop, non-irrigated" + ], + [ + "Transformation, to arable, non-irrigated, diverse-intensive", + "Transformation, to annual crop, non-irrigated, extensive" + ], + [ + "Transformation, to arable, non-irrigated, fallow", + "Transformation, to cropland fallow (non-use)" + ], + [ + "Transformation, to arable, non-irrigated, monotone-intensive", + "Transformation, to annual crop, non-irrigated, intensive" + ], + [ + "Transformation, to forest", + "Transformation, to forest, unspecified" + ], + [ + "Transformation, to pasture and meadow", + "Transformation, to pasture, man made" + ], + [ + "Transformation, to pasture and meadow, extensive", + "Transformation, to pasture, man made, extensive" + ], + [ + "Transformation, to pasture and meadow, intensive", + "Transformation, to pasture, man made, intensive" + ], + [ + "Transformation, to permanent crop, fruit", + "Transformation, to permanent crop, irrigated" + ], + [ + "Transformation, to permanent crop, fruit, extensive", + "Transformation, to permanent crop, irrigated, extensive" + ], + [ + "Transformation, to permanent crop, vine", + "Transformation, to permanent crop, non-irrigated" + ], + [ + "Transformation, to permanent crop, vine, extensive", + "Transformation, to permanent crop, non-irrigated, extensive" + ], + [ + "Transformation, to permanent crop, vine, intensive", + "Transformation, to permanent crop, non-irrigated, intensive" + ], + [ + "Transformation, to traffic area, rail embankment", + "Transformation, to traffic area, rail/road embankment" + ], + [ + "Transformation, to tropical rain forest", + "Transformation, to forest, primary (non-use)" + ], + [ + "Transformation, to unknown", + "Transformation, to unspecified" + ], + [ + "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore, in ground", + "Zinc, Zn 0.63%, in mixed ore, in ground" + ], + [ + "1-Butanol", + "Butanol" + ], + [ + "1-Propanol", + "Propanol" + ], + [ + "Benzene, dichloro-", + "Benzene, dichloro" + ], + [ + "2-Butene, 2-methyl-", + "2-Methyl-2-butene" + ], + [ + "Benzene, 1,2-dichloro-", + "o-Dichlorobenzene" + ], + [ + "Benzene, 1-methyl-2-nitro-", + "o-Nitrotoluene" + ], + [ + "x \"Wat", + "Benz(a)anthracene" + ], + [ + "Benzo(g,h,i)perylene", + "Benzo(ghi)perylene" + ], + [ + "Carbamic acid, butyl-, 3-iodo-2-propynyl ester", + "Butylcarbamate, iodopropynyl" + ], + [ + "Carbon", + "Elemental carbon" + ], + [ + "Carbon monoxide, land transformation", + "Carbon monoxide, from soil or biomass stock" + ], + [ + "Dioxin, 2,3,7,8 Tetrachlorodibenzo-p-", + "Dioxins, measured as 2,3,7,8-tetrachlorodibenzo-p-dioxin" + ], + [ + "Ethane, chloro-", + "Monochloroethane" + ], + [ + "Ethanol, 2-ethoxy-", + "Ethylene glycol monoethyl ether" + ], + [ + "Methane, land transformation", + "Methane, from soil or biomass stock" + ], + [ + "Methane, tetrachloro-, CFC-10", + "Methane, tetrachloro-, R-10" + ], + [ + "Methane, tetrafluoro-, CFC-14", + "Methane, tetrafluoro-, R-14" + ], + [ + "Methylamine", + "Methyl amine" + ], + [ + "Nitric oxide", + "Nitrogen monoxide" + ], + [ + "Parathion, methyl", + "Methyl parathion" + ], + [ + "Pentane, 2,2,4-trimethyl-", + "2,2,4-Trimethyl pentane" + ], + [ + "Pentane, 2-methyl-", + "2-Methyl pentane" + ], + [ + "Phenol, 2,4-dichloro-", + "Phenol, 2,4-dichloro" + ], + [ + "Pyraclostrobin (prop)", + "Pyraclostrobin" + ], + [ + "Quizalofop ethyl ester", + "Quizalofop-ethyl" + ], + [ + "Toluene, 2-chloro-", + "Toluene, 2-chloro" + ], + [ + "2-Methyl-4-chlorophenoxyacetic acid", + "MCPA" + ], + [ + "Benzene, pentachloronitro-", + "Quintozene" + ], + [ + "Chlortoluron", + "Chlorotoluron" + ], + [ + "Dipropylthiocarbamic acid S-ethyl ester", + "EPTC" + ], + [ + "Dithianone", + "Dithianon" + ], + [ + "Iprodione", + "Iprodion" + ], + [ + "Metam-sodium dihydrate", + "Metam-sodium" + ], + [ + "Monosodium acid methanearsonate", + "MSMA" + ], + [ + "Oxydemeton methyl", + "Oxydemeton-methyl" + ], + [ + "Thiazole, 2-(thiocyanatemethylthio)benzo-", + "TCMTB" + ], + [ + "Triallate", + "Tri-allate" + ], + [ + "Acrylate", + "Acrylate, ion" + ], + [ + "Ammonium", + "Ammonium, ion" + ], + [ + "Arsenic", + "Arsenic, ion" + ], + [ + "Benzo(a)anthracene", + "Benz(a)anthracene" + ], + [ + "Cadmium", + "Cadmium, 0.30% in sulfide, Cd 0.18%, Pb, Zn, Ag, In, in ground" + ], + [ + "Chromium", + "Chromium, 25.5% in chromite, 11.6% in crude ore, in ground" + ], + [ + "Copper", + "Copper, ion" + ], + [ + "Formic acid, thallium(1+) salt", + "Formate" + ], + [ + "Hydrogen chloride", + "Hydrochloric acid" + ], + [ + "Iron", + "Iron, 46% in ore, 25% in crude ore, in ground" + ], + [ + "Lithium", + "Lithium, 0.15% in brine, in ground" + ], + [ + "Naphthalene", + "Naphtalene" + ], + [ + "Nickel", + "Nickel, ion" + ], + [ + "Perchlorate", + "Perchlorate, ion" + ], + [ + "Potassium", + "Potassium, ion" + ], + [ + "Silver", + "Silver, ion" + ], + [ + "Sodium", + "Sodium, ion" + ], + [ + "Thiocyanate", + "Thiocyanate, ion" + ], + [ + "Tin", + "Tin, 79% in cassiterite, 0.1% in crude ore, in ground" + ], + [ + "Titanium", + "Titanium, ion" + ], + [ + "Vanadium", + "Vanadium, ion" + ], + [ + "Zinc", + "Zinc, 9.0% in sulfide, Zn 5.3%, Pb, Ag, Cd, In, in ground" + ], + [ + "Sulphur", + "Sulfur" + ], + [ + "Aluminium", + "Aluminium, in ground" + ], + [ + "Anhydrite", + "Anhydrite, in ground" + ], + [ + "Argon", + "Argon-40" + ], + [ + "Barite", + "Barite, 15% in crude ore, in ground" + ], + [ + "Basalt", + "Basalt, in ground" + ], + [ + "Borax", + "Borax, in ground" + ], + [ + "Bromine", + "Bromine, 0.23% in water" + ], + [ + "Calcite", + "Calcite, in ground" + ], + [ + "Cerium", + "Cerium, 24% in bastnasite, 2.4% in crude ore, in ground" + ], + [ + "Chrysotile", + "Chrysotile, in ground" + ], + [ + "Cinnabar", + "Cinnabar, in ground" + ], + [ + "Clay, bentonite", + "Clay, bentonite, in ground" + ], + [ + "Clay, unspecified", + "Clay, unspecified, in ground" + ], + [ + "Coal, brown", + "Coal, brown, in ground" + ], + [ + "Cobalt", + "Cobalt, in ground" + ], + [ + "Colemanite", + "Colemanite, in ground" + ], + [ + "Copper, 0.52% in sulfide, Cu 0.27% and Mo 8.2E-3% in crude ore", + "Copper, 0.52% in sulfide, Cu 0.27% and Mo 8.2E-3% in crude ore, in ground" + ], + [ + "Copper, 0.59% in sulfide, Cu 0.22% and Mo 8.2E-3% in crude ore", + "Copper, 0.59% in sulfide, Cu 0.22% and Mo 8.2E-3% in crude ore, in ground" + ], + [ + "Copper, 0.97% in sulfide, Cu 0.36% and Mo 4.1E-2% in crude ore", + "Copper, 0.97% in sulfide, Cu 0.36% and Mo 4.1E-2% in crude ore, in ground" + ], + [ + "Copper, 0.99% in sulfide, Cu 0.36% and Mo 8.2E-3% in crude ore", + "Copper, 0.99% in sulfide, Cu 0.36% and Mo 8.2E-3% in crude ore, in ground" + ], + [ + "Copper, 1.13% in sulfide, Cu 0.76% and Ni 0.76% in crude ore", + "Copper, 1.13% in sulfide, Cu 0.76% and Ni 0.76% in crude ore, in ground" + ], + [ + "Copper, 1.18% in sulfide, Cu 0.39% and Mo 8.2E-3% in crude ore", + "Copper, 1.18% in sulfide, Cu 0.39% and Mo 8.2E-3% in crude ore, in ground" + ], + [ + "Copper, 1.42% in sulfide, Cu 0.81% and Mo 8.2E-3% in crude ore", + "Copper, 1.42% in sulfide, Cu 0.81% and Mo 8.2E-3% in crude ore, in ground" + ], + [ + "Copper, 2.19% in sulfide, Cu 1.83% and Mo 8.2E-3% in crude ore", + "Copper, 2.19% in sulfide, Cu 1.83% and Mo 8.2E-3% in crude ore, in ground" + ], + [ + "Copper, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore", + "Cu, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore, in ground" + ], + [ + "Copper, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore", + "Cu, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore, in ground" + ], + [ + "Diatomite", + "Diatomite, in ground" + ], + [ + "Dolomite", + "Dolomite, in ground" + ], + [ + "Europium", + "Europium, 0.06% in bastnasite, 0.006% in crude ore, in ground" + ], + [ + "Feldspar", + "Feldspar, in ground" + ], + [ + "Fluorine", + "Fluorine, 4.5% in apatite, 1% in crude ore, in ground" + ], + [ + "Fluorine, 4.5% in apatite, 3% in crude ore", + "Fluorine, 4.5% in apatite, 3% in crude ore, in ground" + ], + [ + "Fluorspar", + "Fluorspar, 92%, in ground" + ], + [ + "Gadolinium", + "Gadolinium, 0.15% in bastnasite, 0.015% in crude ore, in ground" + ], + [ + "Gallium", + "Gallium, in ground" + ], + [ + "Gas, mine, off-gas, process, coal mining/m3", + "Gas, mine, off-gas, process, coal mining" + ], + [ + "Gas, natural/m3", + "Gas, natural, in ground" + ], + [ + "Gold", + "Gold, Au 1.4E-4%, in ore, in ground" + ], + [ + "Gold, Au 1.1E-4%, Ag 4.2E-3%, in ore", + "Gold, Au 1.1E-4%, Ag 4.2E-3%, in ore, in ground" + ], + [ + "Gold, Au 1.3E-4%, Ag 4.6E-5%, in ore", + "Gold, Au 1.3E-4%, Ag 4.6E-5%, in ore, in ground" + ], + [ + "Gold, Au 2.1E-4%, Ag 2.1E-4%, in ore", + "Gold, Au 2.1E-4%, Ag 2.1E-4%, in ore, in ground" + ], + [ + "Gold, Au 4.3E-4%, in ore", + "Gold, Au 4.3E-4%, in ore, in ground" + ], + [ + "Gold, Au 4.9E-5%, in ore", + "Gold, Au 4.9E-5%, in ore, in ground" + ], + [ + "Gold, Au 6.7E-4%, in ore", + "Gold, Au 6.7E-4%, in ore, in ground" + ], + [ + "Gold, Au 7.1E-4%, in ore", + "Gold, Au 7.1E-4%, in ore, in ground" + ], + [ + "Gold, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore", + "Gold, Au 9.7E-4%, in mixed ore, in ground" + ], + [ + "Granite", + "Granite, in ground" + ], + [ + "Gravel", + "Gravel, in ground" + ], + [ + "Gypsum", + "Gypsum, in ground" + ], + [ + "Indium", + "Indium, 0.005% in sulfide, In 0.003%, Pb, Zn, Ag, Cd, in ground" + ], + [ + "Iodine", + "Iodine, 0.03% in water" + ], + [ + "Kaolinite", + "Kaolinite, 24% in crude ore, in ground" + ], + [ + "Kieserite", + "Kieserite, 25% in crude ore, in ground" + ], + [ + "Krypton", + "Krypton, in air" + ], + [ + "Lanthanum", + "Lanthanum, 7.2% in bastnasite, 0.72% in crude ore, in ground" + ], + [ + "Lead", + "Lead, 5.0% in sulfide, Pb 3.0%, Zn, Ag, Cd, In, in ground" + ], + [ + "Lead, Pb 0.014%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, in ore", + "Lead, Pb 0.014%, in mixed ore, in ground" + ], + [ + "Magnesite", + "Magnesite, 60% in crude ore, in ground" + ], + [ + "Magnesium", + "Magnesium, 0.13% in water" + ], + [ + "Manganese", + "Manganese, 35.7% in sedimentary deposit, 14.2% in crude ore, in ground" + ], + [ + "Metamorphous rock, graphite containing", + "Metamorphous rock, graphite containing, in ground" + ], + [ + "Molybdenum", + "Molybdenum, 0.11% in sulfide, Mo 4.1E-2% and Cu 0.36% in crude ore, in ground" + ], + [ + "Molybdenum, 0.010% in sulfide, Mo 8.2E-3% and Cu 1.83% in crude ore", + "Molybdenum, 0.010% in sulfide, Mo 8.2E-3% and Cu 1.83% in crude ore, in ground" + ], + [ + "Molybdenum, 0.014% in sulfide, Mo 8.2E-3% and Cu 0.81% in crude ore", + "Molybdenum, 0.014% in sulfide, Mo 8.2E-3% and Cu 0.81% in crude ore, in ground" + ], + [ + "Molybdenum, 0.016% in sulfide, Mo 8.2E-3% and Cu 0.27% in crude ore", + "Molybdenum, 0.016% in sulfide, Mo 8.2E-3% and Cu 0.27% in crude ore, in ground" + ], + [ + "Molybdenum, 0.022% in sulfide, Mo 8.2E-3% and Cu 0.22% in crude ore", + "Molybdenum, 0.022% in sulfide, Mo 8.2E-3% and Cu 0.22% in crude ore, in ground" + ], + [ + "Molybdenum, 0.022% in sulfide, Mo 8.2E-3% and Cu 0.36% in crude ore", + "Molybdenum, 0.022% in sulfide, Mo 8.2E-3% and Cu 0.36% in crude ore, in ground" + ], + [ + "Molybdenum, 0.025% in sulfide, Mo 8.2E-3% and Cu 0.39% in crude ore", + "Molybdenum, 0.025% in sulfide, Mo 8.2E-3% and Cu 0.39% in crude ore, in ground" + ], + [ + "Neodymium", + "Neodymium, 4% in bastnasite, 0.4% in crude ore, in ground" + ], + [ + "Nickel, 1.13% in sulfide, Ni 0.76% and Cu 0.76% in crude ore", + "Nickel, 1.13% in sulfide, Ni 0.76% and Cu 0.76% in crude ore, in ground" + ], + [ + "Nickel, 1.98% in silicates, 1.04% in crude ore", + "Nickel, 1.98% in silicates, 1.04% in crude ore, in ground" + ], + [ + "Nickel, Ni 2.3E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Cu 3.2E+0% in ore", + "Ni, Ni 2.3E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Cu 3.2E+0% in ore, in ground" + ], + [ + "Nickel, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore", + "Ni, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore, in ground" + ], + [ + "Occupation, arable, greenhouse", + "Occupation, annual crop, greenhouse" + ], + [ + "Occupation, arable, irrigated", + "Occupation, annual crop, irrigated" + ], + [ + "Occupation, arable, irrigated, intensive", + "Occupation, permanent crop, irrigated, intensive" + ], + [ + "Occupation, arable, non-irrigated, extensive", + "Occupation, annual crop, non-irrigated, extensive" + ], + [ + "Occupation, arable, non-irrigated, intensive", + "Occupation, permanent crop, non-irrigated, intensive" + ], + [ + "Occupation, grassland, not used", + "Occupation, grassland, natural (non-use)" + ], + [ + "Occupation, urban/industrial fallow", + "Occupation, urban/industrial fallow (non-use)" + ], + [ + "Oil, crude", + "Oil, crude, in ground" + ], + [ + "Olivine", + "Olivine, in ground" + ], + [ + "Palladium, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore", + "Pd, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore, in ground" + ], + [ + "Palladium, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore", + "Pd, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore, in ground" + ], + [ + "Peat", + "Peat, in ground" + ], + [ + "Perlite", + "Perlite, in ground" + ], + [ + "Phosphorus", + "Phosphorus, 18% in apatite, 12% in crude ore, in ground" + ], + [ + "Phosphorus, 18% in apatite, 4% in crude ore", + "Phosphorus, 18% in apatite, 4% in crude ore, in ground" + ], + [ + "Platinum, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore", + "Pt, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore, in ground" + ], + [ + "Platinum, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore", + "Pt, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore, in ground" + ], + [ + "Potassium chloride", + "Sylvite, 25 % in sylvinite, in ground" + ], + [ + "Praseodymium", + "Praseodymium, 0.42% in bastnasite, 0.042% in crude ore, in ground" + ], + [ + "Pumice", + "Pumice, in ground" + ], + [ + "Pyrite", + "Pyrite, in ground" + ], + [ + "Rhenium", + "Rhenium, in crude ore, in ground" + ], + [ + "Rhodium, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore", + "Rh, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore, in ground" + ], + [ + "Rhodium, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore", + "Rh, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore, in ground" + ], + [ + "Samarium", + "Samarium, 0.3% in bastnasite, 0.03% in crude ore, in ground" + ], + [ + "Sand", + "Sand, unspecified, in ground" + ], + [ + "Shale", + "Shale, in ground" + ], + [ + "Silver, 0.007% in sulfide, Ag 0.004%, Pb, Zn, Cd, In", + "Silver, 0.007% in sulfide, Ag 0.004%, Pb, Zn, Cd, In, in ground" + ], + [ + "Silver, 3.2ppm in sulfide, Ag 1.2ppm, Cu and Te, in crude ore", + "Silver, 3.2ppm in sulfide, Ag 1.2ppm, Cu and Te, in crude ore, in ground" + ], + [ + "Silver, Ag 2.1E-4%, Au 2.1E-4%, in ore", + "Silver, Ag 2.1E-4%, Au 2.1E-4%, in ore, in ground" + ], + [ + "Silver, Ag 4.2E-3%, Au 1.1E-4%, in ore", + "Silver, Ag 4.2E-3%, Au 1.1E-4%, in ore, in ground" + ], + [ + "Silver, Ag 4.6E-5%, Au 1.3E-4%, in ore", + "Silver, Ag 4.6E-5%, Au 1.3E-4%, in ore, in ground" + ], + [ + "Silver, Ag 9.7E-4%, Au 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore", + "Silver, Ag 9.7E-4%, in mixed ore, in ground" + ], + [ + "Sodium chloride", + "Sodium chloride, in ground" + ], + [ + "Sodium nitrate", + "Sodium nitrate, in ground" + ], + [ + "Sodium sulfate", + "Sodium sulphate, various forms, in ground" + ], + [ + "Spodumene", + "Spodumene, in ground" + ], + [ + "Stibnite", + "Stibnite, in ground" + ], + [ + "Sulfur", + "Sulfur, in ground" + ], + [ + "Talc", + "Talc, in ground" + ], + [ + "Tantalum", + "Tantalum, 81.9% in tantalite, 1.6E-4% in crude ore, in ground" + ], + [ + "Tellurium", + "Tellurium, 0.5ppm in sulfide, Te 0.2ppm, Cu and Ag, in crude ore, in ground" + ], + [ + "TiO2, 54% in ilmenite, 18% in crude ore", + "TiO2, 54% in ilmenite, 18% in crude ore, in ground" + ], + [ + "TiO2, 54% in ilmenite, 2.6% in crude ore", + "TiO2, 54% in ilmenite, 2.6% in crude ore, in ground" + ], + [ + "TiO2, 95% in rutile, 0.40% in crude ore", + "TiO2, 95% in rutile, 0.40% in crude ore, in ground" + ], + [ + "Transformation, from arable, greenhouse", + "Transformation, from annual crop, greenhouse" + ], + [ + "Transformation, from arable, irrigated, intensive", + "Transformation, from annual crop, irrigated, intensive" + ], + [ + "Transformation, from arable, non-irrigated, extensive", + "Transformation, from annual crop, non-irrigated, extensive" + ], + [ + "Transformation, from arable, non-irrigated, intensive", + "Transformation, from annual crop, non-irrigated, intensive" + ], + [ + "Transformation, from forest, primary", + "Transformation, from forest, primary (non-use)" + ], + [ + "Transformation, from grassland, not used", + "Transformation, from grassland, natural (non-use)" + ], + [ + "Transformation, from permanent crops, irrigated, intensive", + "Transformation, from permanent crop, irrigated, intensive" + ], + [ + "Transformation, from permanent crops, non-irrigated, intensive", + "Transformation, from permanent crop, non-irrigated, intensive" + ], + [ + "Transformation, to arable, fallow", + "Transformation, to cropland fallow (non-use)" + ], + [ + "Transformation, to arable, greenhouse", + "Transformation, to annual crop, greenhouse" + ], + [ + "Transformation, to arable, irrigated, extensive", + "Transformation, to annual crop, irrigated, extensive" + ], + [ + "Transformation, to arable, irrigated, intensive", + "Transformation, to annual crop, irrigated, intensive" + ], + [ + "Transformation, to arable, non-irrigated, extensive", + "Transformation, to annual crop, non-irrigated, extensive" + ], + [ + "Transformation, to arable, non-irrigated, intensive", + "Transformation, to annual crop, non-irrigated, intensive" + ], + [ + "Transformation, to permanent crops, irrigated, intensive", + "Transformation, to permanent crop, irrigated, intensive" + ], + [ + "Transformation, to permanent crops, non-irrigated", + "Transformation, to permanent crop, non-irrigated" + ], + [ + "Transformation, to permanent crops, non-irrigated, intensive", + "Transformation, to permanent crop, non-irrigated, intensive" + ], + [ + "Transformation, to urban/industrial fallow", + "Transformation, to urban/industrial fallow (non-use)" + ], + [ + "Ulexite", + "Ulexite, in ground" + ], + [ + "Uranium", + "Uranium, in ground" + ], + [ + "Vermiculite", + "Vermiculite, in ground" + ], + [ + "Water, AT", + "Water, unspecified natural origin" + ], + [ + "Water, AU", + "Water, unspecified natural origin" + ], + [ + "Water, BA", + "Water, unspecified natural origin" + ], + [ + "Water, BE", + "Water, unspecified natural origin" + ], + [ + "Water, BG", + "Water, unspecified natural origin" + ], + [ + "Water, BR", + "Water, unspecified natural origin" + ], + [ + "Water, CA", + "Water, unspecified natural origin" + ], + [ + "Water, CH", + "Water, unspecified natural origin" + ], + [ + "Water, CL", + "Water, unspecified natural origin" + ], + [ + "Water, CN", + "Water, unspecified natural origin" + ], + [ + "Water, cooling, unspecified natural origin/m3", + "Water, cooling, unspecified natural origin" + ], + [ + "Water, CZ", + "Water, unspecified natural origin" + ], + [ + "Water, DE", + "Water, unspecified natural origin" + ], + [ + "Water, DK", + "Water, unspecified natural origin" + ], + [ + "Water, ES", + "Water, unspecified natural origin" + ], + [ + "Water, Europe without Switzerland", + "Water, unspecified natural origin" + ], + [ + "Water, FI", + "Water, unspecified natural origin" + ], + [ + "Water, FR", + "Water, unspecified natural origin" + ], + [ + "Water, GB", + "Water, unspecified natural origin" + ], + [ + "Water, GLO", + "Water, unspecified natural origin" + ], + [ + "Water, GR", + "Water, unspecified natural origin" + ], + [ + "Water, HR", + "Water, unspecified natural origin" + ], + [ + "Water, HU", + "Water, unspecified natural origin" + ], + [ + "Water, ID", + "Water, unspecified natural origin" + ], + [ + "Water, IE", + "Water, unspecified natural origin" + ], + [ + "Water, IN", + "Water, unspecified natural origin" + ], + [ + "Water, IR", + "Water, unspecified natural origin" + ], + [ + "Water, IT", + "Water, unspecified natural origin" + ], + [ + "Water, JP", + "Water, unspecified natural origin" + ], + [ + "Water, KR", + "Water, unspecified natural origin" + ], + [ + "Water, LU", + "Water, unspecified natural origin" + ], + [ + "Water, MA", + "Water, unspecified natural origin" + ], + [ + "Water, MX", + "Water, unspecified natural origin" + ], + [ + "Water, MY", + "Water, unspecified natural origin" + ], + [ + "Water, NL", + "Water, unspecified natural origin" + ], + [ + "Water, NO", + "Water, unspecified natural origin" + ], + [ + "Water, NORDEL", + "Water, unspecified natural origin" + ], + [ + "Water, PE", + "Water, unspecified natural origin" + ], + [ + "Water, PG", + "Water, unspecified natural origin" + ], + [ + "Water, PH", + "Water, unspecified natural origin" + ], + [ + "Water, PL", + "Water, unspecified natural origin" + ], + [ + "Water, PT", + "Water, unspecified natural origin" + ], + [ + "Water, RAF", + "Water, unspecified natural origin" + ], + [ + "Water, RAS", + "Water, unspecified natural origin" + ], + [ + "Water, RER", + "Water, unspecified natural origin" + ], + [ + "Water, RLA", + "Water, unspecified natural origin" + ], + [ + "Water, RME", + "Water, unspecified natural origin" + ], + [ + "Water, RNA", + "Water, unspecified natural origin" + ], + [ + "Water, RO", + "Water, unspecified natural origin" + ], + [ + "Water, RoW", + "Water, unspecified natural origin" + ], + [ + "Water, RS", + "Water, unspecified natural origin" + ], + [ + "Water, RU", + "Water, unspecified natural origin" + ], + [ + "Water, SE", + "Water, unspecified natural origin" + ], + [ + "Water, SI", + "Water, unspecified natural origin" + ], + [ + "Water, SK", + "Water, unspecified natural origin" + ], + [ + "Water, TH", + "Water, unspecified natural origin" + ], + [ + "Water, TR", + "Water, unspecified natural origin" + ], + [ + "Water, TW", + "Water, unspecified natural origin" + ], + [ + "Water, TZ", + "Water, unspecified natural origin" + ], + [ + "Water, UA", + "Water, unspecified natural origin" + ], + [ + "Water, UCTE", + "Water, unspecified natural origin" + ], + [ + "Water, UCTE without Germany", + "Water, unspecified natural origin" + ], + [ + "Water, US", + "Water, unspecified natural origin" + ], + [ + "Water, WEU", + "Water, unspecified natural origin" + ], + [ + "Water, ZA", + "Water, unspecified natural origin" + ], + [ + "Water_3.1_m3", + "Water, unspecified natural origin" + ], + [ + "Wood, unspecified, standing/m3", + "Wood, unspecified, standing" + ], + [ + "Xenon", + "Xenon, in air" + ], + [ + "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore", + "Zinc, Zn 0.63%, in mixed ore, in ground" + ], + [ + "Zirconium", + "Zirconium, 50% in zircon, 0.39% in crude ore, in ground" + ], + [ + "Water, unspecified natural origin, BA", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, HR", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, PG", + "Water, unspecified natural origin" + ], + [ + "Water, lake, AT", + "Water, unspecified natural origin" + ], + [ + "Water, lake, BE", + "Water, unspecified natural origin" + ], + [ + "Water, lake, BG", + "Water, unspecified natural origin" + ], + [ + "Water, lake, CA", + "Water, unspecified natural origin" + ], + [ + "Water, lake, CH", + "Water, unspecified natural origin" + ], + [ + "Water, lake, CN", + "Water, unspecified natural origin" + ], + [ + "Water, lake, CZ", + "Water, unspecified natural origin" + ], + [ + "Water, lake, DE", + "Water, unspecified natural origin" + ], + [ + "Water, lake, DK", + "Water, unspecified natural origin" + ], + [ + "Water, lake, ES", + "Water, unspecified natural origin" + ], + [ + "Water, lake, Europe without Switzerland", + "Water, unspecified natural origin" + ], + [ + "Water, lake, FI", + "Water, unspecified natural origin" + ], + [ + "Water, lake, FR", + "Water, unspecified natural origin" + ], + [ + "Water, lake, GB", + "Water, unspecified natural origin" + ], + [ + "Water, lake, GLO", + "Water, unspecified natural origin" + ], + [ + "Water, lake, HU", + "Water, unspecified natural origin" + ], + [ + "Water, lake, IT", + "Water, unspecified natural origin" + ], + [ + "Water, lake, JP", + "Water, unspecified natural origin" + ], + [ + "Water, lake, KR", + "Water, unspecified natural origin" + ], + [ + "Water, lake, LU", + "Water, unspecified natural origin" + ], + [ + "Water, lake, NL", + "Water, unspecified natural origin" + ], + [ + "Water, lake, NO", + "Water, unspecified natural origin" + ], + [ + "Water, lake, PL", + "Water, unspecified natural origin" + ], + [ + "Water, lake, PT", + "Water, unspecified natural origin" + ], + [ + "Water, lake, RER", + "Water, unspecified natural origin" + ], + [ + "Water, lake, RNA", + "Water, unspecified natural origin" + ], + [ + "Water, lake, RoW", + "Water, unspecified natural origin" + ], + [ + "Water, lake, RU", + "Water, unspecified natural origin" + ], + [ + "Water, lake, SE", + "Water, unspecified natural origin" + ], + [ + "Water, lake, SK", + "Water, unspecified natural origin" + ], + [ + "Water, lake, TR", + "Water, unspecified natural origin" + ], + [ + "Water, lake, TW", + "Water, unspecified natural origin" + ], + [ + "Water, lake, US", + "Water, unspecified natural origin" + ], + [ + "Water, river, AT", + "Water, unspecified natural origin" + ], + [ + "Water, river, AU", + "Water, unspecified natural origin" + ], + [ + "Water, river, BE", + "Water, unspecified natural origin" + ], + [ + "Water, river, BG", + "Water, unspecified natural origin" + ], + [ + "Water, river, BR", + "Water, unspecified natural origin" + ], + [ + "Water, river, CA", + "Water, unspecified natural origin" + ], + [ + "Water, river, CH", + "Water, unspecified natural origin" + ], + [ + "Water, river, CN", + "Water, unspecified natural origin" + ], + [ + "Water, river, CZ", + "Water, unspecified natural origin" + ], + [ + "Water, river, DE", + "Water, unspecified natural origin" + ], + [ + "Water, river, DK", + "Water, unspecified natural origin" + ], + [ + "Water, river, ES", + "Water, unspecified natural origin" + ], + [ + "Water, river, Europe without Switzerland", + "Water, unspecified natural origin" + ], + [ + "Water, river, FI", + "Water, unspecified natural origin" + ], + [ + "Water, river, FR", + "Water, unspecified natural origin" + ], + [ + "Water, river, GB", + "Water, unspecified natural origin" + ], + [ + "Water, river, GLO", + "Water, unspecified natural origin" + ], + [ + "Water, river, HU", + "Water, unspecified natural origin" + ], + [ + "Water, river, IN", + "Water, unspecified natural origin" + ], + [ + "Water, river, IT", + "Water, unspecified natural origin" + ], + [ + "Water, river, JP", + "Water, unspecified natural origin" + ], + [ + "Water, river, KR", + "Water, unspecified natural origin" + ], + [ + "Water, river, LU", + "Water, unspecified natural origin" + ], + [ + "Water, river, MY", + "Water, unspecified natural origin" + ], + [ + "Water, river, NL", + "Water, unspecified natural origin" + ], + [ + "Water, river, NO", + "Water, unspecified natural origin" + ], + [ + "Water, river, PE", + "Water, unspecified natural origin" + ], + [ + "Water, river, PH", + "Water, unspecified natural origin" + ], + [ + "Water, river, PL", + "Water, unspecified natural origin" + ], + [ + "Water, river, PT", + "Water, unspecified natural origin" + ], + [ + "Water, river, RAS", + "Water, unspecified natural origin" + ], + [ + "Water, river, RER", + "Water, unspecified natural origin" + ], + [ + "Water, river, RLA", + "Water, unspecified natural origin" + ], + [ + "Water, river, RNA", + "Water, unspecified natural origin" + ], + [ + "Water, river, RoW", + "Water, unspecified natural origin" + ], + [ + "Water, river, RU", + "Water, unspecified natural origin" + ], + [ + "Water, river, SE", + "Water, unspecified natural origin" + ], + [ + "Water, river, SK", + "Water, unspecified natural origin" + ], + [ + "Water, river, TR", + "Water, unspecified natural origin" + ], + [ + "Water, river, TW", + "Water, unspecified natural origin" + ], + [ + "Water, river, TZ", + "Water, unspecified natural origin" + ], + [ + "Water, river, US", + "Water, unspecified natural origin" + ], + [ + "Water, river, WEU", + "Water, unspecified natural origin" + ], + [ + "Water, river, ZA", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, Europe without Switzerland", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, GLO", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, NORDEL", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, RAF", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, RER", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, RNA", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, RoW", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, TW", + "Water, unspecified natural origin" + ], + [ + "Water, unspecified natural origin, WEU", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, AT", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, AU", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, BE", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, BG", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, BR", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, CA", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, CH", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, CN", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, CZ", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, DE", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, DK", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, ES", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, Europe without Switzerland", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, FI", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, FR", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, GB", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, GLO", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, HU", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, ID", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, IN", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, IT", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, JP", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, KR", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, LU", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, MA", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, MY", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, NL", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, NO", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, NORDEL", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, PE", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, PG", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, PH", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, PL", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, PT", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, RER", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, RLA", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, RNA", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, RoW", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, RU", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, SE", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, SK", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, TR", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, TW", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, US", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, WEU", + "Water, unspecified natural origin" + ], + [ + "Water, well, in ground, ZA", + "Water, unspecified natural origin" + ] +] diff --git a/src/flowmapper/data/manual_name_match_simapro_ecoinvent_3.9.json b/src/flowmapper/data/manual_name_match_simapro_ecoinvent_3.9.json new file mode 100644 index 0000000..72f192c --- /dev/null +++ b/src/flowmapper/data/manual_name_match_simapro_ecoinvent_3.9.json @@ -0,0 +1,610 @@ +[ + [ + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, freezing {GLO}| market for transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refri(...)_16 | Cut-off", + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, freezing {GLO}| transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, freezing | Cut-off" + ], + [ + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, cooling {GLO}| market for transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrig(...)_8 | Cut-off", + "Transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, cooling {GLO}| transport, freight, lorry with refrigeration machine, 7.5-16 ton, EURO5, carbon dioxide, liquid refrigerant, cooling | Cut-off" + ], + [ + "Occupation, agriculture", + "Occupation, annual crop" + ], + [ + "Occupation, dump site, benthos", + "Occupation, seabed, unspecified" + ], + [ + "Occupation, forest, intensive, normal", + "Occupation, forest, intensive" + ], + [ + "Occupation, forest, intensive, short-cycle", + "Occupation, forest, intensive" + ], + [ + "Occupation, forest, used", + "Occupation, forest, intensive" + ], + [ + "Occupation, grassland/pasture/meadow", + "Occupation, pasture, man made" + ], + [ + "Occupation, industrial area, benthos", + "Occupation, seabed, unspecified" + ], + [ + "Occupation, industrial area, built up", + "Occupation, industrial area" + ], + [ + "Occupation, industrial area, vegetation", + "Occupation, industrial area" + ], + [ + "Occupation, permanent crop, fruit, intensive", + "Occupation, permanent crop, irrigated" + ], + [ + "Occupation, sea and ocean", + "Occupation, seabed, unspecified" + ], + [ + "Occupation, sea and ocean", + "Occupation, seabed, unspecified" + ], + [ + "Occupation, traffic area", + "Occupation, traffic area, road network" + ], + [ + "Occupation, traffic area, road embankment", + "Occupation, traffic area, road network" + ], + [ + "Occupation, unspecified, used", + "Occupation, unspecified" + ], + [ + "Occupation, water bodies, artificial", + "Occupation, lake, artificial" + ], + [ + "Occupation, rivers, artificial", + "Occupation, river, artificial" + ], + [ + "Occupation, lakes, artificial", + "Occupation, lake, artificial" + ], + [ + "Occupation, water courses, artificial", + "Occupation, river, artificial" + ], + [ + "Occupation, wetland", + "Occupation, inland waterbody, unspecified" + ], + [ + "Transformation, from agriculture", + "Transformation, from annual crop" + ], + [ + "Transformation, from annual crop, non-irrigated, fallow", + "Transformation, from pasture, man made" + ], + [ + "Transformation, from forest, intensive, clear-cutting", + "Transformation, from forest, intensive" + ], + [ + "Transformation, from forest, used", + "Transformation, from forest, intensive" + ], + [ + "Transformation, from grassland", + "Transformation, from grassland, natural (non-use)" + ], + [ + "Transformation, from grassland/pasture/meadow", + "Transformation, from pasture, man made" + ], + [ + "Transformation, from industrial area, benthos", + "Transformation, from seabed, unspecified" + ], + [ + "Transformation, from industrial area, built up", + "Transformation, from industrial area" + ], + [ + "Transformation, from industrial area, vegetation", + "Transformation, from industrial area" + ], + [ + "Transformation, from permanent crop, fruit", + "Transformation, from permanent crop, irrigated" + ], + [ + "Transformation, from tropical rain forest", + "Transformation, from forest, extensive" + ], + [ + "Transformation, from unspecified, used", + "Transformation, from unspecified" + ], + [ + "Transformation, to agriculture", + "Transformation, to annual crop" + ], + [ + "Transformation, to annual crop, fallow", + "Transformation, to arable land, unspecified use" + ], + [ + "Transformation, to annual crop, non-irrigated, fallow", + "Transformation, to annual crop, non-irrigated, extensive" + ], + [ + "Transformation, to dump site, benthos", + "Transformation, to seabed, unspecified" + ], + [ + "Transformation, to forest, intensive, clear-cutting", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to forest, intensive, normal", + "Transformation, to forest, intensive" + ], + [ + "Transformation, from arable, organic", + "Transformation, from arable land, unspecified use" + ], + [ + "Transformation, to forest, intensive, short-cycle", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to forest, used", + "Transformation, to forest, intensive" + ], + [ + "Transformation, to grassland/pasture/meadow", + "Transformation, to pasture, man made" + ], + [ + "Transformation, to industrial area, benthos", + "Transformation, to seabed, unspecified" + ], + [ + "Transformation, to industrial area, built up", + "Transformation, to industrial area" + ], + [ + "Transformation, to industrial area, vegetation", + "Transformation, to industrial area" + ], + [ + "Transformation, to permanent crop, fruit, intensive", + "Transformation, to permanent crop, irrigated" + ], + [ + "Transformation, to sea and ocean", + "Transformation, to seabed, unspecified" + ], + [ + "Transformation, to traffic area, road embankment", + "Transformation, to traffic area, road network" + ], + [ + "Transformation, to unspecified, used", + "Transformation, to unspecified" + ], + [ + "Transformation, to urban/industrial fallow", + "Transformation, to industrial area" + ], + [ + "Transformation, to lakes, artificial", + "Transformation, to lake, artificial" + ], + [ + "Transformation, to rivers, artificial", + "Transformation, to river, artificial" + ], + [ + "Transformation, to water bodies, artificial", + "Transformation, to river, artificial" + ], + [ + "Transformation, to water courses, artificial", + "Transformation, to river, artificial" + ], + [ + "Particulates, < 10 um", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, < 10 um (mobile)", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, < 10 um (stationary)", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, > 10 um (process)", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, SPM", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, diesel soot", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Particulates, unspecified", + "Particulates, > 2.5 um, and < 10um" + ], + [ + "Water/m3", + "Water" + ], + [ + "Sulfur monoxide", + "Sulfur oxides" + ], + [ + "Nitrogen, atmospheric", + "Nitrogen" + ], + [ + "Phenols, unspecified", + "Phenol" + ], + [ + "Ammonia, as N", + "Ammonia" + ], + [ + "AOX (Adsorbable Organic Halogens)", + "AOX, Adsorbable Organic Halogen as Cl" + ], + [ + "AOX, Adsorbable Organic Halogen", + "AOX, Adsorbable Organic Halogen as Cl" + ], + [ + "Alkane (unspecified)", + "Hydrocarbons, aliphatic, alkanes, unspecified" + ], + [ + "Antimony, ion", + "Antimony" + ], + [ + "Formetanate", + "Formetanate hydrochloride" + ], + [ + "Haloxyfop", + "haloxyfop-ethoxyethyl" + ], + [ + "BOD5 (Biological Oxygen Demand)", + "BOD5, Biological Oxygen Demand" + ], + [ + "COD (Chemical Oxygen Demand)", + "COD, Chemical Oxygen Demand" + ], + [ + "Bauxite", + "Gangue" + ], + [ + "Copper ore", + "Copper, Cu 0.2%, in mixed ore" + ], + [ + "Copper, Cu 0.38%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Pb 0.014%, in ore", + "Copper, Cu 0.38%, in mixed ore" + ], + [ + "Copper, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore", + "Copper, Cu 0.38%, in mixed ore" + ], + [ + "Copper, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore", + "Copper, 0.52% in sulfide, Cu 0.27% and Mo 8.2E-3% in crude ore" + ], + [ + "Coal, 18 MJ per kg", + "Coal, hard, unspecified" + ], + [ + "Coal, brown, 10 MJ per kg", + "Coal, brown" + ], + [ + "Coal, brown, 8 MJ per kg", + "Coal, brown" + ], + [ + "Coal, hard", + "Coal, hard, unspecified" + ], + [ + "Crude oil", + "Oil, crude" + ], + [ + "Energy, from biomass", + "Energy, gross calorific value, in biomass" + ], + [ + "Energy, from coal", + "Energy, gross calorific value, in biomass" + ], + [ + "Gas, mine, off-gas, process, coal mining/kg", + "Gas, mine, off-gas, process, coal mining" + ], + [ + "Antimony, ion", + "Antimony" + ], + [ + "Arsenic (V)", + "Arsenic" + ], + [ + "Barium (II)", + "Barium" + ], + [ + "Beryllium (II)", + "Beryllium" + ], + [ + "Cadmium (II)", + "Cadmium" + ], + [ + "Calcium", + "Calcium, ion" + ], + [ + "Chromium (III)", + "Chromium III" + ], + [ + "Chromium (VI)", + "Chromium VI" + ], + [ + "Chromium, ion", + "Chromium" + ], + [ + "Copper, ion", + "Copper" + ], + [ + "Formetanate", + "Formetanate hydrochloride" + ], + [ + "Haloxyfop", + "haloxyfop-ethoxyethyl" + ], + [ + "Lead (II)", + "Lead" + ], + [ + "Mercury (II)", + "Mercury" + ], + [ + "Manganese (II)", + "Manganese" + ], + [ + "Molybdenum (VI)", + "Molybdenum" + ], + [ + "Nickel (II)", + "Nickel" + ], + [ + "Oils, biogenic", + "Oils, non-fossil" + ], + [ + "Particulates, > 10 um", + "Particulate Matter, > 10 um" + ], + [ + "Plutonium", + "Plutonium-238" + ], + [ + "Potassium (I)", + "Potassium" + ], + [ + "Ruthenium", + "Ruthenium-103" + ], + [ + "Silver (I)", + "Silver" + ], + [ + "Thallium (I)", + "Thallium" + ], + [ + "Zinc (II)", + "Zinc" + ], + [ + "Ethene", + "Ethylene" + ], + [ + "Water, process, unspecified natural origin/m3", + "Water, unspecified natural origin" + ], + [ + "Water, cooling, unspecified natural origin/m3", + "Water, cooling, unspecified natural origin" + ], + [ + "Water, unspecified natural origin/m3", + "Water, unspecified natural origin" + ], + [ + "Carbon", + "Elemental carbon" + ], + [ + "Radioactive species, unspecified", + "Radioactive species, Nuclides, unspecified" + ], + [ + "Methane, tetrachloro-, CFC-10", + "Methane, tetrachloro-, R-10" + ], + [ + "Methane, tetrachloro-, CFC-14", + "Methane, tetrachloro-, R-14" + ], + [ + "Cesium-134", + "Caesium-134" + ], + [ + "Cesium-136", + "Caesium-136" + ], + [ + "Cesium", + "Caesium" + ], + [ + "Cesium-137", + "Caesium-137" + ], + [ + "Particulates, < 2.5 um", + "Particulate Matter, < 2.5 um" + ], + [ + "Silver, Ag 9.7E-4%, Au 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore", + "Silver, Ag 9.7E-4%, in mixed ore" + ], + [ + "1-Butanol", + "Butanol" + ], + [ + "AOX, Adsorbable Organic Halogen as Cl", + "AOX, Adsorbable Organic Halogen" + ], + [ + "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore", + "Zinc, Zn 0.63%, in mixed ore" + ], + [ + "Transformation, to urban", + "Transformation, to urban, continuously built" + ], + [ + "Occupation, urban", + "Occupation, urban, continuously built" + ], + [ + "VOC, volatile organic compounds, unspecified origin", + "VOC, volatile organic compounds" + ], + [ + "Lead, Pb 0.014%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, in ore", + "Lead, Pb 0.014%, in mixed ore" + ], + [ + "Particulates, < 10 um (stationary)", + "Particulate Matter, > 2.5 um and < 10um" + ], + [ + "Nickel, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore", + "Nickel, Ni 2.5E+0%, in mixed ore" + ], + [ + "Platinum, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore", + "Pt, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + ], + [ + "Occupation, annual crop, non-irrigated, diverse-intensive", + "Occupation, annual crop, non-irrigated, intensive" + ], + [ + "Gold, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore", + "Gold, Au 9.7E-4%, in mixed ore" + ], + [ + "Particulates, > 2.5 um, and < 10um", + "Particulate Matter, > 2.5 um and < 10um" + ], + [ + "Platinum, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore", + "Pt, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + ], + [ + "Sodium sulfate", + "Sodium sulphate, various forms" + ], + [ + "Transformation, from pasture and meadow, organic", + "Transformation, from pasture, man made, extensive" + ], + [ + "Palladium, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore", + "Pd, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + ], + [ + "Clay", + "Clay, bentonite" + ], + [ + "NMVOC, non-methane volatile organic compounds, unspecified origin", + "NMVOC, non-methane volatile organic compounds" + ], + [ + "Rhodium, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore", + "Rh, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + ], + [ + "Rhodium, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore", + "Rh, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + ], + [ + "Palladium, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore", + "Pd, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + ], + [ + "Energy, gross calorific value, in biomass, resource correction", + "Energy, gross calorific value, in biomass" + ], + [ + "Transformation, to arable, organic", + "Transformation, to arable land, unspecified use" + ] +] diff --git a/src/flowmapper/data/names_and_locations.json b/src/flowmapper/data/names_and_locations.json new file mode 100644 index 0000000..c6a2c63 --- /dev/null +++ b/src/flowmapper/data/names_and_locations.json @@ -0,0 +1,28 @@ +[ + { + "source": "water, well", + "target": "water, well, in ground" + }, + { + "source": "water/m3", + "target": "water" + }, + { + "source": "water, unspecified natural origin/m3", + "target": "water, unspecified natural origin" + }, + { + "source": "water (ersatz), net cons., irrigation", + "target": "water, unspecified natural origin", + "extra": { + "irrigation": true + } + }, + { + "source": "water (ersatz), net cons., non-irrigation", + "target": "water, unspecified natural origin", + "extra": { + "irrigation": false + } + } +] diff --git a/src/flowmapper/data/places.json b/src/flowmapper/data/places.json new file mode 100644 index 0000000..50aad67 --- /dev/null +++ b/src/flowmapper/data/places.json @@ -0,0 +1,578 @@ +[ + "AD", + "AE", + "AF", + "AG", + "AI", + "Akrotiri", + "AL", + "AM", + "AO", + "AQ", + "AR", + "AS", + "ASCC", + "Ashmore and Cartier Islands", + "Asia without China", + "AT", + "AU", + "AU-AC", + "AU-ACT", + "AU-IOT", + "AU-JBT", + "AU-NSW", + "AU-NT", + "AU-QLD", + "AU-SA", + "AU-TAS", + "AU-VIC", + "AU-WA", + "AUS-AC", + "AUS-ACT", + "AUS-IOT", + "AUS-NSW", + "AUS-NTR", + "AUS-QNS", + "AUS-SAS", + "AUS-TSM", + "AUS-VCT", + "AUS-WAS", + "Australia, including overseas territories", + "AW", + "AX", + "AZ", + "BA", + "Bajo Nuevo", + "BALTSO", + "BB", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BL", + "BM", + "BN", + "BO", + "BQ", + "BR", + "BR-AC", + "BR-AL", + "BR-AM", + "BR-AP", + "BR-BA", + "BR-CE", + "BR-DF", + "BR-ES", + "BR-GO", + "BR-MA", + "BR-MG", + "BR-Mid-western grid", + "BR-MS", + "BR-MT", + "BR-North-eastern grid", + "BR-Northern grid", + "BR-PA", + "BR-PB", + "BR-PE", + "BR-PI", + "BR-PR", + "BR-RJ", + "BR-RN", + "BR-RO", + "BR-RR", + "BR-RS", + "BR-SC", + "BR-SE", + "BR-South-eastern grid", + "BR-South-eastern/Mid-western grid", + "BR-Southern grid", + "BR-SP", + "BR-TO", + "BS", + "BT", + "BV", + "BW", + "BY", + "BZ", + "CA", + "CA-AB", + "CA-BC", + "CA-MB", + "CA-NB", + "CA-NF", + "CA-NS", + "CA-NT", + "CA-NU", + "CA-ON", + "CA-PE", + "CA-QC", + "CA-SK", + "CA-YK", + "Canada without Alberta and Quebec", + "Canada without Alberta", + "Canada without Quebec", + "Canary Islands", + "CC", + "CD", + "Central Asia", + "CENTREL", + "CF", + "CG", + "CH", + "China w/o Inner Mongol", + "CI", + "CK", + "CL", + "Clipperton Island", + "CM", + "CN", + "CN-AH", + "CN-BJ", + "CN-CCG", + "CN-CQ", + "CN-CSG", + "CN-ECGC", + "CN-FJ", + "CN-GD", + "CN-GS", + "CN-GX", + "CN-GZ", + "CN-HA", + "CN-HB", + "CN-HE", + "CN-HL", + "CN-HN", + "CN-HU", + "CN-JL", + "CN-JS", + "CN-JX", + "CN-LN", + "CN-NCGC", + "CN-NECG", + "CN-NM", + "CN-NWG", + "CN-NX", + "CN-QH", + "CN-SA", + "CN-SC", + "CN-SD", + "CN-SGCC", + "CN-SH", + "CN-SWG", + "CN-SX", + "CN-TJ", + "CN-XJ", + "CN-XZ", + "CN-YN", + "CN-ZJ", + "CO", + "Coral Sea Islands", + "CR", + "Crimea", + "CS", + "CSG", + "CU", + "CUSMA/T-MEC/USMCA", + "CV", + "CW", + "CX", + "CY", + "Cyprus No Mans Area", + "CZ", + "DE", + "Dhekelia Base", + "DJ", + "DK", + "DM", + "DO", + "DZ", + "EC", + "EE", + "EEU", + "EG", + "EH", + "ENTSO-E", + "ER", + "ES", + "ET", + "Europe without Austria", + "Europe without NORDEL (NCPA)", + "Europe without Switzerland and Austria", + "Europe without Switzerland and France", + "Europe without Switzerland", + "Europe, without Russia and T\u00fcrkiye", + "Europe, without Russia and Turkey", + "FI", + "FJ", + "FK", + "FM", + "FO", + "FR", + "France, including overseas territories", + "FRCC", + "FSU", + "GA", + "GB", + "GD", + "GE", + "GF", + "GG", + "GH", + "GI", + "GL", + "GLO", + "GM", + "GN", + "GP", + "GQ", + "GR", + "GS", + "GT", + "GU", + "Guantanamo Bay", + "GW", + "GY", + "HICC", + "HK", + "HM", + "HN", + "HR", + "HT", + "HU", + "IAI Area, Africa", + "IAI Area, Asia, without China and GCC", + "IAI Area, EU27 & EFTA", + "IAI Area, Gulf Cooperation Council", + "IAI Area, North America", + "IAI Area, North America, without Quebec", + "IAI Area, Russia & RER w/o EU27 & EFTA", + "IAI Area, South America", + "IAI Area, West Europe", + "ID", + "IE", + "IL", + "IM", + "IN", + "IN-AN", + "IN-AP", + "IN-AR", + "IN-AS", + "IN-BR", + "IN-CH", + "IN-CT", + "IN-DD", + "IN-DH", + "IN-DL", + "IN-DN", + "IN-Eastern grid", + "IN-GA", + "IN-GJ", + "IN-HP", + "IN-HR", + "IN-Islands", + "IN-JH", + "IN-JK", + "IN-KA", + "IN-KL", + "IN-LA", + "IN-LD", + "IN-MH", + "IN-ML", + "IN-MN", + "IN-MP", + "IN-MZ", + "IN-NL", + "IN-North-eastern grid", + "IN-Northern grid", + "IN-OR", + "IN-PB", + "IN-PY", + "IN-RJ", + "IN-SK", + "IN-Southern grid", + "IN-TG", + "IN-TN", + "IN-TR", + "IN-UP", + "IN-UT", + "IN-WB", + "IN-Western grid", + "Indian Ocean Territories", + "IO", + "IQ", + "IR", + "IS", + "IT", + "JE", + "JM", + "JO", + "JP", + "KE", + "KG", + "KH", + "KI", + "KM", + "KN", + "KP", + "KR", + "KW", + "KY", + "KZ", + "LA", + "LB", + "LC", + "LI", + "LK", + "LR", + "LS", + "LT", + "LU", + "LV", + "LY", + "MA", + "MC", + "MD", + "ME", + "MF", + "MG", + "MH", + "MK", + "ML", + "MM", + "MN", + "MO", + "MP", + "MQ", + "MR", + "MRO", + "MRO, US only", + "MS", + "MT", + "MU", + "MV", + "MW", + "MX", + "MY", + "MZ", + "NA", + "NAFTA", + "NC", + "NE", + "NF", + "NG", + "NI", + "NL", + "NO", + "NORDEL", + "North America without Quebec", + "Northern Cyprus", + "NP", + "NPCC", + "NPCC, US only", + "NR", + "NU", + "NZ", + "OCE", + "OM", + "PA", + "PE", + "PF", + "PG", + "PH", + "PK", + "PL", + "PM", + "PN", + "PR", + "PS", + "PT", + "PW", + "PY", + "QA", + "Qu\u00e9bec, HQ distribution network", + "RAF", + "RAS", + "RE", + "RER w/o AT+BE+CH+DE+FR+IT", + "RER w/o CH+DE", + "RER w/o DE+NL+NO", + "RER w/o DE+NL+NO+RU", + "RER w/o DE+NL+RU", + "RER w/o RU", + "RER", + "RFC", + "RLA", + "RME", + "RNA", + "RO", + "RoE", + "RoW", + "RS", + "RU", + "Russia (Asia)", + "Russia (Europe)", + "RW", + "SA", + "SAS", + "SB", + "SC", + "Scarborough Reef", + "SD", + "SE", + "SERC", + "Serranilla Bank", + "SG", + "SGCC", + "SH", + "SI", + "Siachen Glacier", + "SJ", + "SK", + "SL", + "SM", + "SN", + "SO", + "Somaliland", + "SPP", + "Spratly Islands", + "SR", + "SS", + "ST", + "SV", + "SX", + "SY", + "SZ", + "TC", + "TD", + "TF", + "TG", + "TH", + "TJ", + "TK", + "TL", + "TM", + "TN", + "TO", + "TR", + "TRE", + "TT", + "TV", + "TW", + "TZ", + "UA", + "UCTE without France", + "UCTE without Germany and France", + "UCTE without Germany", + "UCTE", + "UG", + "UM", + "UN-AMERICAS", + "UN-ASIA", + "UN-AUSTRALIANZ", + "UN-CAMERICA", + "UN-CARIBBEAN", + "UN-EAFRICA", + "UN-EASIA", + "UN-EEUROPE", + "UN-EUROPE", + "UN-MAFRICA", + "UN-MELANESIA", + "UN-MICRONESIA", + "UN-NAFRICA", + "UN-NEUROPE", + "UN-OCEANIA", + "UN-POLYNESIA", + "UN-SAMERICA", + "UN-SASIA", + "UN-SEASIA", + "UN-SEUROPE", + "UN-WAFRICA", + "UN-WASIA", + "United States, including overseas territories", + "US", + "US-AK", + "US-AL", + "US-AR", + "US-ASCC", + "US-AZ", + "US-CA", + "US-CO", + "US-CT", + "US-DC", + "US-DE", + "US-FL", + "US-FRCC", + "US-GA", + "US-HI", + "US-HICC", + "US-IA", + "US-ID", + "US-IL", + "US-IN", + "US-KS", + "US-KY", + "US-LA", + "US-MA", + "US-MD", + "US-ME", + "US-MI", + "US-MN", + "US-MO", + "US-MRO", + "US-MS", + "US-MT", + "US-NC", + "US-ND", + "US-NE", + "US-NH", + "US-NJ", + "US-NM", + "US-NPCC", + "US-NV", + "US-NY", + "US-OH", + "US-OK", + "US-OR", + "US-PA", + "US-PR", + "US-RFC", + "US-RI", + "US-SC", + "US-SD", + "US-SERC", + "US-SPP", + "US-TN", + "US-TRE", + "US-TX", + "US-UT", + "US-VA", + "US-VT", + "US-WA", + "US-WECC", + "US-WI", + "US-WV", + "US-WY", + "UY", + "UZ", + "VA", + "VC", + "VE", + "VG", + "VI", + "VN", + "VU", + "WECC", + "WECC, US only", + "WEU", + "WF", + "WS", + "XK", + "YE", + "YT", + "ZA", + "ZM", + "ZW" +] diff --git a/src/flowmapper/data/simapro-2025-ecoinvent-3-contexts.json b/src/flowmapper/data/simapro-2025-ecoinvent-3-contexts.json new file mode 100644 index 0000000..738c131 --- /dev/null +++ b/src/flowmapper/data/simapro-2025-ecoinvent-3-contexts.json @@ -0,0 +1,711 @@ +{ + "name": "SimaPro-2025-ecoinvent-3.12-context", + "licenses": [ + { + "name": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "title": "Creative Commons Attribution 4.0 International" + } + ], + "version": "1.0.0", + "description": "Context mapping from 2025 SimaPro to ecoinvent 3.12", + "case-insensitive": true, + "created": "2025-11-10T12:34:56Z", + "contributors": [ + { + "title": "Chris Mutel", + "path": "https://chris.mutel.org/", + "roles": ["author"] + } + ], + "graph_context": [ + "nodes" + ], + "mapping": { + "source": { + "expression language": "JSONPath", + "labels": { + "context": "$.context" + } + }, + "target": { + "expression language": "JSONPath", + "labels": { + "context": "$.context" + } + } + }, + "update": [ + { + "source": { + "context": "air/unspecified" + }, + "target": { + "context": [ + "air", + "unspecified" + ] + } + }, + { + "source": { + "context": "air/high. pop." + }, + "target": { + "context": [ + "air", + "urban air close to ground" + ] + } + }, + { + "source": { + "context": "air/indoor" + }, + "target": { + "context": [ + "air", + "indoor" + ] + } + }, + { + "source": { + "context": "air/low. pop." + }, + "target": { + "context": [ + "air", + "non-urban air or from high stacks" + ] + } + }, + { + "source": { + "context": "air/low. pop., long-term" + }, + "target": { + "context": [ + "air", + "low population density, long-term" + ] + } + }, + { + "source": { + "context": "air/stratosphere + troposphere" + }, + "target": { + "context": [ + "air", + "lower stratosphere + upper troposphere" + ] + } + }, + { + "source": { + "context": "Emissions to air" + }, + "target": { + "context": [ + "air", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to air/" + }, + "target": { + "context": [ + "air", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to air/unspecified" + }, + "target": { + "context": [ + "air", + "unspecified" + ] + } + }, + { + "source": { + "context": "Airborne emissions/(unspecified)" + }, + "target": { + "context": [ + "air", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to air/high. pop." + }, + "target": { + "context": [ + "air", + "urban air close to ground" + ] + } + }, + { + "source": { + "context": "Airborne emissions/high. pop." + }, + "target": { + "context": [ + "air", + "urban air close to ground" + ] + } + }, + { + "source": { + "context": "Emissions to air/low. pop." + }, + "target": { + "context": [ + "air", + "non-urban air or from high stacks" + ] + } + }, + { + "source": { + "context": "Airborne emissions/low. pop." + }, + "target": { + "context": [ + "air", + "non-urban air or from high stacks" + ] + } + }, + { + "source": { + "context": "Emissions to air/indoor" + }, + "target": { + "context": [ + "air", + "non-urban air or from high stacks" + ] + } + }, + { + "source": { + "context": "Airborne emissions/indoor" + }, + "target": { + "context": [ + "air", + "non-urban air or from high stacks" + ] + } + }, + { + "source": { + "context": "Emissions to air/low. pop., long-term" + }, + "target": { + "context": [ + "air", + "low population density, long-term" + ] + } + }, + { + "source": { + "context": "Airborne emissions/low. pop., long-term" + }, + "target": { + "context": [ + "air", + "low population density, long-term" + ] + } + }, + { + "source": { + "context": "Emissions to air/stratosphere + troposphere" + }, + "target": { + "context": [ + "air", + "lower stratosphere + upper troposphere" + ] + } + }, + { + "source": { + "context": "Airborne emissions/stratosphere + troposphere" + }, + "target": { + "context": [ + "air", + "lower stratosphere + upper troposphere" + ] + } + }, + { + "source": { + "context": "Emissions to soil/unspecified" + }, + "target": { + "context": [ + "soil", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to soil/(unspecified)" + }, + "target": { + "context": [ + "soil", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to soil" + }, + "target": { + "context": [ + "soil", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to soil/" + }, + "target": { + "context": [ + "soil", + "unspecified" + ] + } + }, + { + "source": { + "context": "Emissions to soil/agricultural" + }, + "target": { + "context": [ + "soil", + "agricultural" + ] + } + }, + { + "source": { + "context": "Emissions to soil/forestry" + }, + "target": { + "context": [ + "soil", + "forestry" + ] + } + }, + { + "source": { + "context": "Emissions to soil/industrial" + }, + "target": { + "context": [ + "soil", + "industrial" + ] + } + }, + { + "source": { + "context": "Final waste flows/(unspecified)" + }, + "target": { + "context": [ + "inventory indicator", + "waste" + ] + } + }, + { + "source": { + "context": "Final waste flows/" + }, + "target": { + "context": [ + "inventory indicator", + "waste" + ] + } + }, + { + "source": { + "context": "water/unspecified" + }, + "target": { + "context": [ + "water", + "unspecified" + ] + } + }, + { + "source": { + "context": "water/groundwater" + }, + "target": { + "context": [ + "water", + "ground-" + ] + } + }, + { + "source": { + "context": "water/groundwater, long-term" + }, + "target": { + "context": [ + "water", + "ground-, long-term" + ] + } + }, + { + "source": { + "context": "water/fossilwater" + }, + "target": { + "context": [ + "water", + "ground-, long-term" + ] + } + }, + { + "source": { + "context": "water/river" + }, + "target": { + "context": [ + "water", + "surface water" + ] + } + }, + { + "source": { + "context": "emissions to water" + }, + "target": { + "context": [ + "water", + "unspecified" + ] + } + }, + { + "source": { + "context": "emissions to water/" + }, + "target": { + "context": [ + "water", + "unspecified" + ] + } + }, + { + "source": { + "context": "emissions to water/unspecified" + }, + "target": { + "context": [ + "water", + "unspecified" + ] + } + }, + { + "source": { + "context": "Waterborne emissions/(unspecified)" + }, + "target": { + "context": [ + "water", + "unspecified" + ] + } + }, + { + "source": { + "context": "emissions to water/groundwater" + }, + "target": { + "context": [ + "water", + "ground-" + ] + } + }, + { + "source": { + "context": "emissions to water/groundwater, long-term" + }, + "target": { + "context": [ + "water", + "ground-, long-term" + ] + } + }, + { + "source": { + "context": "Waterborne emissions/groundwater, long-term" + }, + "target": { + "context": [ + "water", + "ground-, long-term" + ] + } + }, + { + "source": { + "context": "emissions to water/lake" + }, + "target": { + "context": [ + "water", + "surface water" + ] + } + }, + { + "source": { + "context": "emissions to water/ocean" + }, + "target": { + "context": [ + "water", + "ocean" + ] + } + }, + { + "source": { + "context": "Waterborne emissions/ocean" + }, + "target": { + "context": [ + "water", + "ocean" + ] + } + }, + { + "source": { + "context": "emissions to water/river" + }, + "target": { + "context": [ + "water", + "surface water" + ] + } + }, + { + "source": { + "context": "Waterborne emissions/river" + }, + "target": { + "context": [ + "water", + "surface water" + ] + } + }, + { + "source": { + "context": "emissions to water/river, long-term" + }, + "target": { + "context": [ + "water", + "surface water" + ] + } + }, + { + "source": { + "context": "resources/biotic" + }, + "target": { + "context": [ + "natural resource", + "biotic" + ] + } + }, + { + "source": { + "context": "resources/in air" + }, + "target": { + "context": [ + "natural resource", + "in air" + ] + } + }, + { + "source": { + "context": "resources/in ground" + }, + "target": { + "context": [ + "natural resource", + "in ground" + ] + } + }, + { + "source": { + "context": "resources/in water" + }, + "target": { + "context": [ + "natural resource", + "in water" + ] + } + }, + { + "source": { + "context": "resources/land" + }, + "target": { + "context": [ + "natural resource", + "land" + ] + } + }, + { + "source": { + "context": "Raw" + }, + "target": { + "context": [ + "natural resource" + ] + }, + "comment": "Dummy value used for matching; not a real ecoinvent context" + }, + { + "source": { + "context": "Raw/" + }, + "target": { + "context": [ + "natural resource" + ] + }, + "comment": "Dummy value used for matching; not a real ecoinvent context" + }, + { + "source": { + "context": "Raw materials/" + }, + "target": { + "context": [ + "natural resource" + ] + }, + "comment": "Dummy value used for matching; not a real ecoinvent context" + }, + { + "source": { + "context": "Raw/biotic" + }, + "target": { + "context": [ + "natural resource", + "biotic" + ] + } + }, + { + "source": { + "context": "Raw/in air" + }, + "target": { + "context": [ + "natural resource", + "in air" + ] + } + }, + { + "source": { + "context": "Raw/in ground" + }, + "target": { + "context": [ + "natural resource", + "in ground" + ] + } + }, + { + "source": { + "context": "Raw/land" + }, + "target": { + "context": [ + "natural resource", + "land" + ] + } + }, + { + "source": { + "context": "Raw materials/land" + }, + "target": { + "context": [ + "natural resource", + "land" + ] + } + }, + { + "source": { + "context": "Raw/in water" + }, + "target": { + "context": [ + "natural resource", + "in water" + ] + } + } + ] +} diff --git a/src/flowmapper/data/standard-units-harmonization.json b/src/flowmapper/data/standard-units-harmonization.json new file mode 100644 index 0000000..caa8d44 --- /dev/null +++ b/src/flowmapper/data/standard-units-harmonization.json @@ -0,0 +1,488 @@ +{ + "name": "Flowmapper-standard-units-harmonization", + "licenses": [ + { + "name": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "title": "Creative Commons Attribution 4.0 International" + } + ], + "version": "1.0.0", + "description": "Standard flowmapper unit harmonization linked with Pint customization", + "created": "2025-11-10T12:34:56Z", + "case-insensitive": true, + "contributors": [ + { + "title": "Chris Mutel", + "path": "https://chris.mutel.org/", + "roles": ["author"] + } + ], + "graph_context": [ + "nodes" + ], + "mapping": { + "source": { + "expression language": "JSONPath", + "labels": { + "unit": "$.unit" + } + }, + "target": { + "expression language": "JSONPath", + "labels": { + "unit": "$.unit" + } + } + }, + "update": [ + { + "source": { + "unit": "a" + }, + "target": { + "unit": "year" + } + }, + { + "source": { + "unit": "bq" + }, + "target": { + "unit": "becquerel" + } + }, + { + "source": { + "unit": "g" + }, + "target": { + "unit": "gram" + } + }, + { + "source": { + "unit": "gj" + }, + "target": { + "unit": "gigajoule" + } + }, + { + "source": { + "unit": "GJ" + }, + "target": { + "unit": "gigajoule" + } + }, + { + "source": { + "unit": "h" + }, + "target": { + "unit": "hour" + } + }, + { + "source": { + "unit": "ha" + }, + "target": { + "unit": "hectare" + } + }, + { + "source": { + "unit": "hr" + }, + "target": { + "unit": "hour" + } + }, + { + "source": { + "unit": "kbq" + }, + "target": { + "unit": "kilobecquerel" + } + }, + { + "source": { + "unit": "livestock unit" + }, + "target": { + "unit": "livestock_unit" + } + }, + { + "source": { + "unit": "kilowatt hour" + }, + "target": { + "unit": "kilowatt_hour" + } + }, + { + "source": { + "unit": "kBq" + }, + "target": { + "unit": "kilobecquerel" + } + }, + { + "source": { + "unit": "kilo becquerel" + }, + "target": { + "unit": "kilobecquerel" + } + }, + { + "source": { + "unit": "kg" + }, + "target": { + "unit": "kilogram" + } + }, + { + "source": { + "unit": "kgkm" + }, + "target": { + "unit": "kilogram_kilometer" + } + }, + { + "source": { + "unit": "km" + }, + "target": { + "unit": "kilometer" + } + }, + { + "source": { + "unit": "kj" + }, + "target": { + "unit": "kilojoule" + } + }, + { + "source": { + "unit": "kJ" + }, + "target": { + "unit": "kilojoule" + } + }, + { + "source": { + "unit": "kwh" + }, + "target": { + "unit": "kilowatt hour" + } + }, + { + "source": { + "unit": "l" + }, + "target": { + "unit": "litre" + } + }, + { + "source": { + "unit": "lu" + }, + "target": { + "unit": "livestock unit" + } + }, + { + "source": { + "unit": "m" + }, + "target": { + "unit": "meter" + } + }, + { + "source": { + "unit": "m*year" + }, + "target": { + "unit": "meter_year" + } + }, + { + "source": { + "unit": "m2" + }, + "target": { + "unit": "square_meter" + } + }, + { + "source": { + "unit": "m2*year" + }, + "target": { + "unit": "square_meter_year" + } + }, + { + "source": { + "unit": "square meter-year" + }, + "target": { + "unit": "square_meter_year" + } + }, + { + "source": { + "unit": "cubic meter-year" + }, + "target": { + "unit": "cubic_meter_year" + } + }, + { + "source": { + "unit": "m2a" + }, + "target": { + "unit": "square_meter_year" + } + }, + { + "source": { + "unit": "m2y" + }, + "target": { + "unit": "square_meter_year" + } + }, + { + "source": { + "unit": "cubic meter" + }, + "target": { + "unit": "cubic_meter" + } + }, + { + "source": { + "unit": "m3" + }, + "target": { + "unit": "cubic_meter" + } + }, + { + "source": { + "unit": "m3*year" + }, + "target": { + "unit": "cubic_meter_year" + } + }, + { + "source": { + "unit": "m3a" + }, + "target": { + "unit": "cubic_meter_year" + } + }, + { + "source": { + "unit": "m3y" + }, + "target": { + "unit": "cubic_meter_year" + } + }, + { + "source": { + "unit": "ma" + }, + "target": { + "unit": "meter_year" + } + }, + { + "source": { + "unit": "metric ton*km" + }, + "target": { + "unit": "ton_kilometer" + } + }, + { + "source": { + "unit": "tkm" + }, + "target": { + "unit": "ton_kilometer" + } + }, + { + "source": { + "unit": "mj" + }, + "target": { + "unit": "megajoule" + } + }, + { + "source": { + "unit": "MJ" + }, + "target": { + "unit": "megajoule" + } + }, + { + "source": { + "unit": "my" + }, + "target": { + "unit": "meter_year" + } + }, + { + "source": { + "unit": "standard cubic meter" + }, + "target": { + "unit": "standard_cubic_meter" + } + }, + { + "source": { + "unit": "sm3" + }, + "target": { + "unit": "standard_cubic_meter" + } + }, + { + "source": { + "unit": "normal cubic meter" + }, + "target": { + "unit": "normal_cubic_meter" + } + }, + { + "source": { + "unit": "nm3" + }, + "target": { + "unit": "normal_cubic_meter" + } + }, + { + "source": { + "unit": "sM3" + }, + "target": { + "unit": "standard_cubic_meter" + } + }, + { + "source": { + "unit": "nM3" + }, + "target": { + "unit": "normal_cubic_meter" + } + }, + { + "source": { + "unit": "p" + }, + "target": { + "unit": "unit" + } + }, + { + "source": { + "unit": "personkm" + }, + "target": { + "unit": "person_kilometer" + } + }, + { + "source": { + "unit": "person*km" + }, + "target": { + "unit": "person_kilometer" + } + }, + { + "source": { + "unit": "pkm" + }, + "target": { + "unit": "person_kilometer" + } + }, + { + "source": { + "unit": "vkm" + }, + "target": { + "unit": "vehicle_kilometer" + } + }, + { + "source": { + "unit": "kg sw" + }, + "target": { + "unit": "separative_work_unit" + } + }, + { + "source": { + "unit": "swu" + }, + "target": { + "unit": "separative_work_unit" + } + }, + { + "source": { + "unit": "km*year" + }, + "target": { + "unit": "kilometer_year" + } + }, + { + "source": { + "unit": "wh" + }, + "target": { + "unit": "watt_hour" + } + }, + { + "source": { + "unit": "eur2005" + }, + "target": { + "unit": "eur_2005" + } + } + ] +} diff --git a/src/flowmapper/data/units.txt b/src/flowmapper/data/units.txt new file mode 100644 index 0000000..d9b14e3 --- /dev/null +++ b/src/flowmapper/data/units.txt @@ -0,0 +1,43 @@ +# Random +# Pint has the abbreviation Bq capitalized +@alias becquerel = bq + +# Land use +square_meter = meter ** 2 = m2 +cubic_meter = meter ** 3 = m3 +square_meter_year = m2 * year = m2y = m2a +cubic_meter_year = m3 * year = m3y = m3a + +# Gas volume at given conditions +# https://en.wikipedia.org/wiki/Standard_temperature_and_pressure +# 273.15 K (0 °C) and an absolute pressure of exactly 1 bar (100 kPa) +standard_cubic_meter = 44.095 * mole = sm3 +# https://www.sciencedirect.com/topics/engineering/cubic-metre +# There are multiple definitions for this but as we only care about natural gas, using the +# Gas Industry Standards Board seems reasonable. +# 288.15 K (15 °C) and an absolute pressure of exactly 1 atm (101.325 kPa) +# See also https://github.com/qudt/qudt-public-repo/issues/1227 +normal_cubic_meter = 41.739 * mole = nm3 + +# Livestock +livestock_unit = [livestock] = LU + +# Separative work: https://en.wikipedia.org/wiki/Separative_work_units +separative_work = [enrichment] = SW +separative_work_unit = kilogram * SW = SWU + +# Freight +[freight] = [mass] * [length] +ton_kilometer = metric_ton * kilometer = tkm + +# Vehicular travel +vehicle = [vehicular_travel] +vehicle_kilometer = vehicle * kilometer = vkm + +# Personal travel +person = [personal_travel] +person_kilometer = person * kilometer = pkm + +# Currency +eur = [currency] +eur_2005 = 1 * eur diff --git a/src/flowmapper/domain.py b/src/flowmapper/domain.py new file mode 100644 index 0000000..5a51086 --- /dev/null +++ b/src/flowmapper/domain.py @@ -0,0 +1,281 @@ +import itertools +from copy import copy +from dataclasses import asdict, dataclass, field +from enum import StrEnum +from typing import Any, Self + +from six.moves import UserString + +from flowmapper.cas import CASField +from flowmapper.context import ContextField +from flowmapper.location import split_location_suffix +from flowmapper.oxidation_state import OxidationState +from flowmapper.string_field import StringField +from flowmapper.unit import UnitField +from flowmapper.utils import remove_unit_slash + +global_counter = itertools.count(0) + + +@dataclass(frozen=True) +class Flow: + name: StringField + unit: UnitField + context: ContextField + identifier: str | None = None # Internal ID, not necessarily present or unique... + location: str | None = None + oxidation_state: OxidationState | None = None + cas_number: CASField | None = None + synonyms: list[str] = field(default_factory=lambda: []) + _id: int = field(default_factory=lambda: next(global_counter)) + + @staticmethod + def randonneur_mapping() -> dict: + return { + "expression language": "JSONPath", + "labels": { + "unit": "$.unit", + "name": "$.name", + "context": "$.context", + "identifier": "$.identifier", + "location": "$.location", + "cas_number": "$.cas_number", + "synonyms": "$.synonyms", + }, + } + + @classmethod + def from_dict(cls, data: dict) -> Self: + return cls( + name=StringField(data["name"]), + unit=UnitField(data["unit"]), + context=ContextField(data["context"]), + identifier=data.get("identifier"), + location=data.get("location") or None, + oxidation_state=( + OxidationState(data["oxidation_state"]) + if data.get("oxidation_state") + else None + ), + cas_number=CASField.from_string(data.get("cas_number") or None), + synonyms=data.get("synonyms") or [], + ) + + def to_dict(self) -> dict: + data = { + "name": self.name.data, + "unit": self.unit.data, + "context": self.context.as_tuple(), + "identifier": self.identifier, + } + for key in ("location", "oxidation_state", "cas_number", "synonyms"): + if getattr(self, key): + data[key] = getattr(self, key) + return data + + def normalize(self) -> Self: + location, oxidation_state = None, None + name = remove_unit_slash(self) + name, location = split_location_suffix(name) + if OxidationState.has_oxidation_state(name): + oxidation_state, name = OxidationState.from_string(name) + + return type(self)( + identifier=self.identifier, + name=StringField(name).normalize(), + location=location, + oxidation_state=oxidation_state, + unit=self.unit.normalize(), + context=self.context.normalize(), + cas_number=self.cas_number, + synonyms=self.synonyms, + ) + + def __repr__(self) -> str: + return f"""Flow dataclass: + Identifier: {self.identifier} + Name: {self.name} + Context: {self.context} + Unit: {self.unit}""" + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Flow): + return False + return self._id == other._id + + def __lt__(self, other: Self) -> bool: + if not isinstance(other, Flow): + return False + else: + return ( + self.name.data, + self.unit.data, + self.context.value, + self.identifier, + ) < ( + other.name.data, + other.unit.data, + other.context.value, + other.identifier, + ) + + +@dataclass +class NormalizedFlow: + original: Flow + normalized: Flow + current: Flow + matched: bool = False + + @property + def name(self) -> str: + return self.current.name.data + + @property + def unit(self) -> str: + return self.current.unit.data + + @property + def context(self) -> str | list[str] | tuple[str]: + return self.current.context.value + + @property + def identifier(self) -> str | None: + return self.current.identifier + + @property + def location(self) -> str | None: + return self.current.location + + @property + def oxidation_state(self) -> int | None: + return ( + self.current.oxidation_state.value if self.current.oxidation_state else None + ) + + @property + def cas_number(self) -> str | None: + return self.current.cas_number.data if self.current.cas_number else None + + @property + def synonyms(self) -> list[str] | None: + return self.current.synonyms + + def reset_current(self) -> None: + self.current = copy(self.normalized) + + def update_current(self, **kwargs) -> None: + data = self.normalized.to_dict() + data.update(kwargs) + self.current = Flow.from_dict(data) + + @staticmethod + def from_dict(data: dict) -> "NormalizedFlow": + original = Flow.from_dict(data) + # Do data preprocessing here + normalized = original.normalize() + return NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + def unit_compatible(self, other: Self) -> bool: + return self.current.unit.compatible(other.current.unit) + + def conversion_factor(self, other: Self) -> float: + return self.current.unit.conversion_factor(other.current.unit) + + def export(self) -> dict: + data = [ + ("name", self.original.name.data), + ("unit", self.original.unit.data), + ("context", self.original.context.value), + ("identifier", self.original.identifier), + ("location", self.original.location), + ( + "cas_number", + ( + self.normalized.cas_number.export() + if self.normalized.cas_number + else None + ), + ), + ] + return {k: v for k, v in data if v} + + +class MatchCondition(StrEnum): + exact = "http://www.w3.org/2004/02/skos/core#exactMatch" + close = "http://www.w3.org/2004/02/skos/core#closeMatch" + related = "http://www.w3.org/2004/02/skos/core#relatedMatch" + # A triple skos:broader asserts that , the object of the triple, is a broader concept + # than , the subject of the triple. + narrow = "http://www.w3.org/2004/02/skos/core#narrowMatch" # in SKOS the *target* is narrower than the *source* + broad = "http://www.w3.org/2004/02/skos/core#broadMatch" # in SKOS the *target* is broader than the *source* + + def as_glad(self) -> str: + if self.value == "http://www.w3.org/2004/02/skos/core#exactMatch": + return "=" + elif self.value == "http://www.w3.org/2004/02/skos/core#closeMatch": + return "~" + elif self.value == "http://www.w3.org/2004/02/skos/core#relatedMatch": + return "~" + elif self.value == "http://www.w3.org/2004/02/skos/core#narrowMatch": + return ">" + elif self.value == "http://www.w3.org/2004/02/skos/core#broadMatch": + return "<" + raise ValueError # Just for silly type checking + + +@dataclass +class Match: + source: Flow + target: Flow + function_name: str + condition: MatchCondition + conversion_factor: float = 1.0 + comment: str = field(default_factory=lambda: "") + + def export(self, flowmapper_metadata: bool = False) -> dict: + from flowmapper import __version__ + + def serializable(obj: Any) -> Any: + if isinstance(obj, UserString): + return str(obj) + elif isinstance(obj, ContextField): + return obj.value + return obj + + data = asdict(self) + data["source"] = { + k: serializable(v) + for k, v in data["source"].items() + if v and not k.startswith("_") + } + data["target"] = { + k: serializable(v) + for k, v in data["target"].items() + if v and not k.startswith("_") + } + data["condition"] = str(data["condition"]) + + function_name = data.pop("function_name") + if flowmapper_metadata: + data["flowmapper_metadata"] = { + "version": __version__, + "function_name": function_name, + } + + return data + + def __lt__(self, other: "Match") -> bool: + return ( + self.source.name, + self.source.context, + self.target.name, + self.target.context, + ) < ( + other.source.name, + other.source.context, + other.target.name, + other.target.context, + ) diff --git a/src/flowmapper/errors.py b/src/flowmapper/errors.py new file mode 100644 index 0000000..6f512ca --- /dev/null +++ b/src/flowmapper/errors.py @@ -0,0 +1,6 @@ +class DifferingMatches(Exception): + """Multiple different matches given for same flow""" + + +class DifferingConversions(Exception): + """Multiple, different conversion factors provided for a given match""" diff --git a/src/flowmapper/extraction/__init__.py b/src/flowmapper/extraction/__init__.py new file mode 100644 index 0000000..ca59247 --- /dev/null +++ b/src/flowmapper/extraction/__init__.py @@ -0,0 +1,6 @@ +# from flowmapper.extraction.ecoinvent import ecoinvent_biosphere_extractor +from flowmapper.extraction.ecospold2 import ecospold2_biosphere_extractor +from flowmapper.extraction.simapro_csv import simapro_csv_biosphere_extractor +from flowmapper.extraction.simapro_ecospold1 import ( + simapro_ecospold1_biosphere_extractor, +) diff --git a/src/flowmapper/extraction/ecospold2.py b/src/flowmapper/extraction/ecospold2.py new file mode 100644 index 0000000..9489a84 --- /dev/null +++ b/src/flowmapper/extraction/ecospold2.py @@ -0,0 +1,66 @@ +import json +from collections import defaultdict +from pathlib import Path + +import xmltodict + + +def reformat(obj: dict) -> dict: + data = { + "identifier": obj["@id"], + "unit": obj["unitName"]["#text"], + "context": [ + obj["compartment"]["compartment"]["#text"], + obj["compartment"]["subcompartment"]["#text"], + ], + "name": obj["name"]["#text"], + } + if obj.get("synonym") and isinstance(obj["synonym"], list): + data["synonyms"] = [s["#text"] for s in obj["synonym"] if "#text" in s] + elif obj.get("synonym") and "#text" in obj["synonym"]: + data["synonyms"] = [obj["synonym"]["#text"]] + if "@casNumber" in obj: + data["cas_number"] = obj["@casNumber"] + return data + + +def remove_conflicting_synonyms(data: list[dict]) -> list[dict]: + """ + Remove synonyms which conflict with the base names of other flows within the same category tree + branch. + + For example, if flow `A` has a synonym `water` and a context `['ground']`, and flow `B` has the + name `water` and the context `['ground', 'deep']`, then the synonym `water` would be removed + from `A`. However, if `B` had the context `['something', 'else']`, then `water` would be kept, + as it wouldn't directly overlap a different flow in the same category tree branch. + """ + base_names = defaultdict(list) + for obj in data: + if obj.get("name") and obj.get("context"): + base_names[obj["context"][0]].append(obj["name"].lower()) + + for obj in data: + if not (obj.get("synonyms") and obj.get("context")): + continue + obj["synonyms"] = [ + syn + for syn in obj["synonyms"] + if syn.lower() not in base_names[obj["context"][0]] + ] + + return data + + +def ecospold2_biosphere_extractor(input_path: Path, output_path: Path) -> None: + if not input_path.name == "ElementaryExchanges.xml": + raise ValueError("`input_path` must be for a `ElementaryExchanges.xml` file") + with open(input_path) as fs: + ei_xml = xmltodict.parse(fs.read(), strip_whitespace=False)[ + "validElementaryExchanges" + ]["elementaryExchange"] + + with open(output_path, "w") as fs: + json.dump( + remove_conflicting_synonyms([reformat(obj) for obj in ei_xml]), fs, indent=2 + ) + return True diff --git a/src/flowmapper/extraction/simapro_csv.py b/src/flowmapper/extraction/simapro_csv.py new file mode 100644 index 0000000..d6bd383 --- /dev/null +++ b/src/flowmapper/extraction/simapro_csv.py @@ -0,0 +1,62 @@ +import json +from pathlib import Path + +import bw_simapro_csv +import structlog + +logger = structlog.get_logger("filemapper") + + +def is_simapro_csv_file(fp: Path) -> bool: + if not fp.is_file() or not fp.suffix.lower() == ".csv": + return False + try: + bw_simapro_csv.header.parse_header(open(fp, encoding="sloppy-windows-1252"))[ + 0 + ].project + return True + except: + logger.critical( + "Skipping file %s as we can't read it as a SimaPro file", fp.name + ) + return False + + +def simapro_csv_biosphere_extractor(input_path: Path, output_path: Path) -> None: + """Load all simapro files in directory `dirpath`, and extract all biosphere flows""" + if input_path.is_dir(): + simapro_files = [ + fp for fp in sorted(input_path.iterdir()) if is_simapro_csv_file(fp) + ] + elif input_path.is_file(): + simapro_files = [input_path] + else: + raise ValueError + + flows = set() + + for fp in simapro_files: + sp = bw_simapro_csv.SimaProCSV( + fp, stderr_logs=False, write_logs=False, copy_logs=False + ) + for process in filter( + lambda x: isinstance(x, bw_simapro_csv.blocks.Process), + sp.blocks, + ): + for block in filter( + lambda x: isinstance( + x, bw_simapro_csv.blocks.GenericUncertainBiosphere + ), + process.blocks.values(), + ): + for line in block.parsed: + # Restore context to single string as this is expected in our mapping + flows.add(("/".join(line["context"]), line["name"], line["unit"])) + + with open(output_path, "w") as f: + json.dump( + [{"context": c, "name": n, "unit": u} for c, n, u in sorted(flows)], + f, + indent=2, + ensure_ascii=False, + ) diff --git a/src/flowmapper/extraction/simapro_ecospold1.py b/src/flowmapper/extraction/simapro_ecospold1.py new file mode 100644 index 0000000..224e704 --- /dev/null +++ b/src/flowmapper/extraction/simapro_ecospold1.py @@ -0,0 +1,23 @@ +import json +from pathlib import Path + +import pyecospold + + +def simapro_ecospold1_biosphere_extractor(dirpath: Path, output_fp: Path) -> None: + """Load all simapro files in directory `dirpath`, and extract all biosphere flows""" + flows = set() + + for _, es in pyecospold.parse_directory_v1(dirpath): + for ds in es.datasets: + for exc in ds.flowData: + if exc.groupsStr[0] in ("ToNature", "FromNature"): + flows.add(((exc.category, exc.subCategory), exc.name, exc.unit)) + + with open(output_fp, "w") as f: + json.dump( + [{"context": c, "name": n, "unit": u} for c, n, u in sorted(flows)], + f, + indent=2, + ensure_ascii=False, + ) diff --git a/src/flowmapper/flowmap.py b/src/flowmapper/flowmap.py new file mode 100644 index 0000000..009034f --- /dev/null +++ b/src/flowmapper/flowmap.py @@ -0,0 +1,341 @@ +from collections import Counter +from collections.abc import Callable +from functools import cached_property +from pathlib import Path +from time import time + +import pandas as pd +import randonneur +from structlog import get_logger + +from flowmapper import __version__ +from flowmapper.domain import Match, NormalizedFlow +from flowmapper.match import match_rules + +logger = get_logger("flowmapper") + + +class Flowmap: + """ + Crosswalk of flows from a source flow list to a target flow list. + + This class provides functionalities to map flows between different flow lists using a series of predefined match rules. + + Attributes + ---------- + source_flows : list[Flow] + The list of (unique) source flows to be mapped. + source_flows_nomatch : list[Flow] + The list of (unique) source flows that do not match any rule. + target_flows : list[Flow] + The list of target flows for mapping. + target_flows_nomatch : list[Flow] + The list of target flows that do not match any rule. + + """ + + def __init__( + self, + source_flows: list[NormalizedFlow], + target_flows: list[NormalizedFlow], + rules: list[Callable[..., list[Match]]] | None = None, + show_progressbar: bool = True, + ): + """ + Initializes the Flowmap with source and target flows, along with optional matching rules. + + Duplicated flows are removed from both source and targets lists. + + Parameters + ---------- + source_flows : list[Flow] + The list of source flows to be mapped. + target_flows : list[Flow] + The list of target flows for mapping. + rules : list[Callable[..., bool]], optional + Custom rules for matching source flows to target flows. Default is the set of rules defined in `match_rules`. + show_progressbar : bool, optional + If False, progress bar display during the mapping process is disabled. + + """ + self.show_progressbar = show_progressbar + self.rules = rules if rules else match_rules() + + self.source_flows = source_flows + self.target_flows = target_flows + self.matches = [] + + def generate_matches(self) -> None: + """Generate matches by applying match rules""" + for rule in self.rules: + start = time() + result = rule( + source_flows=[flow for flow in self.source_flows if not flow.matched], + target_flows=self.target_flows, + ) + elapsed = time() - start + logger.info( + f"Match function {rule.__name__} produced {len(result)} matches and took {elapsed:.3} seconds." + ) + self.matches.extend(result) + + def matched_source(self): + """ + Provides a list of source flows that have been successfully matched to target flows. + + Returns + ------- + list[Flow] + A list of matched source flow objects. + + """ + result = [ + flow + for flow in self.source_flows + if flow.id in self._matched_source_flows_ids + ] + return result + + @cached_property + def unmatched_source(self): + """ + Provides a list of source flows that have not been matched to any target flows. + + Returns + ------- + list[Flow] + A list of unmatched source flow objects. + + """ + result = [ + flow + for flow in self.source_flows + if flow.id not in self._matched_source_flows_ids + ] + return result + + def matched_source_statistics(self): + """ + Calculates statistics for matched source flows, including the number of matches and the matching percentage for each context. + + Returns + ------- + pandas.DataFrame + A DataFrame containing matching statistics for source flows. + + """ + matched = pd.Series( + Counter([flow.source.context.value for flow in self.matches]) + ).reset_index() + matched.columns = ["context", "matched"] + + total = pd.Series( + Counter([flow.original.context.value for flow in self.source_flows]) + ).reset_index() + total.columns = ["context", "total"] + + df = pd.merge(matched, total, on="context", how="outer") + df = df.fillna(0).astype({"matched": "int", "total": "int"}) + + df["percent"] = df.matched / df.total + result = df.sort_values("percent") + return result + + @cached_property + def matched_target_statistics(self): + """ + Calculates statistics for matched target flows, including the number of matches and the matching percentage for each context. + + Returns + ------- + pandas.DataFrame + A DataFrame containing matching statistics for target flows. + + """ + matched = pd.Series( + Counter([flow.target.context.value for flow in self.matches]) + ).reset_index() + matched.columns = ["context", "matched"] + + total = pd.Series( + Counter([flow.original.context.value for flow in self.target_flows]) + ).reset_index() + total.columns = ["context", "total"] + + df = pd.merge(matched, total, on="context", how="outer") + df = df.fillna(0).astype({"matched": "int", "total": "int"}) + + df["percent"] = df.matched / df.total + result = df.sort_values("percent") + return result + + def print_statistics(self): + """ + Prints out summary statistics for the flow mapping process. + + """ + cardinalities = dict(Counter([x["cardinality"] for x in self.cardinalities()])) + print( + f"""{len(self.source_flows)} source and {len(self.target_flows)} target flows. +{len(self.matches)} mappings ({len(self.matches) / len(self.source_flows):.2%} of total). +Mappings cardinalities: {str(cardinalities)}""" + ) + + def cardinalities(self): + """ + Calculates and returns the cardinalities of mappings between source and target flows. + + Returns + ------- + list[dict] + A sorted list of dictionaries, each indicating the cardinality relationship between a pair of source and target flows. + + """ + mappings = [(match.source._id, match.target._id) for match in self.matches] + lhs_counts = Counter([pair[0] for pair in mappings]) + rhs_counts = Counter([pair[1] for pair in mappings]) + + result = [] + + for lhs, rhs in mappings: + lhs_count = lhs_counts[lhs] + rhs_count = rhs_counts[rhs] + if lhs_count == 1 and rhs_count == 1: + result.append({"from": lhs, "to": rhs, "cardinality": "1:1"}) + elif lhs_count == 1 and rhs_count > 1: + result.append({"from": lhs, "to": rhs, "cardinality": "N:1"}) + elif lhs_count > 1 and rhs_count == 1: + result.append({"from": lhs, "to": rhs, "cardinality": "1:N"}) + elif lhs_count > 1 and rhs_count > 1: + result.append({"from": lhs, "to": rhs, "cardinality": "N:M"}) + + return sorted(result, key=lambda x: x["from"]) + + def to_randonneur( + self, + source_id: str, + target_id: str, + contributors: list, + mapping_source: dict, + mapping_target: dict, + version: str = "1.0.0", + licenses: list | None = None, + homepage: str | None = None, + name: str | None = None, + path: Path | None = None, + ) -> randonneur.Datapackage: + """ + Export mappings using randonneur data migration file format. + + Parameters + ---------- + path : Path, optional + If provided export the output file to disk. + + Returns + ------- + randonneur.Datapackage object. + + """ + dp = randonneur.Datapackage( + name=name or f"{source_id}-{target_id}", + source_id=source_id, + target_id=target_id, + description=f"Flowmapper {__version__} elementary flow correspondence from {source_id} to {target_id}", + contributors=contributors, + mapping_source=mapping_source, + mapping_target=mapping_target, + homepage=homepage, + version=version, + licenses=licenses, + ) + + dp.add_data(verb="update", data=[match.export() for match in self.matches]) + + if path is not None: + dp.to_json(path) + return dp + + def to_glad( + self, + path: Path | None = None, + ensure_id: bool = False, + missing_source: bool = False, + ): + """ + Export mappings using GLAD flow mapping format, optionally ensuring each flow has an identifier. + + Formats the mapping results according to Global LCA Data Access (GLAD) network initiative flow mapping format. + + Parameters + ---------- + path : Path, optional + If provided export the output file to disk. + ensure_id : bool, optional + If True, ensures each flow has an identifier, default is False. + + Returns + ------- + pandas.DataFrame + A DataFrame containing the formatted mapping results in GLAD format. + + """ + data = [] + for match in self.matches: + data.append( + { + "SourceFlowName": str(match.source.name), + "SourceFlowUUID": match.source.identifier + or ("" if ensure_id else None), + "SourceFlowContext": match.source.context.export_as_string( + join_character="/" + ), + "SourceUnit": str(match.source.unit), + "MatchCondition": match.condition.as_glad(), + "ConversionFactor": match.conversion_factor, + "TargetFlowName": str(match.target.name), + "TargetFlowUUID": match.target.identifier + or ("" if ensure_id else None), + "TargetFlowContext": match.target.context.export_as_string( + join_character="/" + ), + "TargetUnit": str(match.target.unit), + "MemoMapper": match.comment, + } + ) + + if missing_source: + for flow_obj in filter(lambda x: not x.matched, self.source_flows): + data.append( + { + "SourceFlowName": str(flow_obj.original.name), + "SourceFlowUUID": flow_obj.original.identifier + or ("" if ensure_id else None), + "SourceFlowContext": flow_obj.original.context.export_as_string(), + "SourceUnit": str(flow_obj.original.unit), + } + ) + + result = pd.DataFrame(data) + + if not path: + return result + else: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + writer = pd.ExcelWriter( + path, + engine="xlsxwriter", + engine_kwargs={"options": {"strings_to_formulas": False}}, + ) + result.to_excel(writer, sheet_name="Mapping", index=False, na_rep="NaN") + + for column in result: + column_length = max( + result[column].astype(str).map(len).max(), len(column) + ) + col_idx = result.columns.get_loc(column) + writer.sheets["Mapping"].set_column(col_idx, col_idx, column_length) + + writer.close() diff --git a/src/flowmapper/location.py b/src/flowmapper/location.py new file mode 100644 index 0000000..4bbeca7 --- /dev/null +++ b/src/flowmapper/location.py @@ -0,0 +1,50 @@ +import importlib.resources as resource +import json +import re +from pathlib import Path + +import structlog + +logger = structlog.get_logger("flowmapper") + +RESULTS_DIR = Path(__file__).parent / "manual_matching" / "results" + +with resource.as_file( + resource.files("flowmapper") / "data" / "places.json" +) as filepath: + places = json.load(open(filepath)) + +ends_with_location = re.compile( + r"(?{})\s*$".format( + "|".join([re.escape(string) for string in places]) + ), +) +# All solutions I found for returning original string instead of +# lower case one were very ugly +# location_reverser = {obj.lower(): obj for obj in places} +# if len(location_reverser) != len(places): +# raise ValueError("Multiple possible locations after lower case conversion") + + +# us_lci_ends_with_location = re.compile( +# "/(?P{})$".format( +# "|".join( +# [ +# re.escape(string) +# for string in places +# if 2 <= len(string) <= 3 and string.upper() == string +# ] +# ) +# ), +# ) + +with resource.as_file( + resource.files("flowmapper") / "data" / "names_and_locations.json" +) as filepath: + names_and_locations = {o["source"]: o for o in json.load(open(filepath))} + + +def split_location_suffix(string: str) -> tuple[str, str | None]: + if match := ends_with_location.search(string): + return string[: match.start()], match.group("location") + return string, None diff --git a/src/flowmapper/main.py b/src/flowmapper/main.py new file mode 100644 index 0000000..5d69d3d --- /dev/null +++ b/src/flowmapper/main.py @@ -0,0 +1,112 @@ +import json +import logging +from copy import copy +from pathlib import Path + +from randonneur import Datapackage +from randonneur_data import Registry + +from flowmapper.domain import Flow, NormalizedFlow +from flowmapper.flowmap import Flowmap +from flowmapper.utils import randonneur_as_function + +logger = logging.getLogger(__name__) + + +def sorting_function(obj: dict) -> tuple: + return ( + obj.get("name", "ZZZ"), + str(obj.get("context", "ZZZ")), + obj.get("unit", "ZZZ"), + ) + + +def flowmapper( + source: Path, + target: Path, + source_id: str, + target_id: str, + contributors: list, + output_dir: Path, + version: str = "1.0.0", + transformations: list[Datapackage | str] | None = None, + unit_normalization: bool = True, + licenses: list | None = None, + homepage: str | None = None, + name: str | None = None, + registry: Registry | None = None, +) -> Flowmap: + """ + Generate mappings between elementary flows lists + """ + output_dir.mkdir(parents=True, exist_ok=True) + transformation_functions = [] + + if transformations is None: + transformations = [] + + if unit_normalization: + transformations.append("Flowmapper-standard-units-harmonization") + + for obj in transformations: + transformation_functions.append( + randonneur_as_function(datapackage=obj, registry=registry) + ) + + original_source_flows = [Flow.from_dict(obj) for obj in json.load(open(source))] + processed_source_flows = [obj.to_dict() for obj in original_source_flows] + original_target_flows = [Flow.from_dict(obj) for obj in json.load(open(target))] + processed_target_flows = [obj.to_dict() for obj in original_target_flows] + + for function in transformation_functions: + processed_source_flows = function(graph=processed_source_flows) + for function in transformation_functions: + processed_target_flows = function(graph=processed_target_flows) + + normalized_source_flows = [ + Flow.from_dict(obj).normalize() for obj in processed_source_flows + ] + normalized_target_flows = [ + Flow.from_dict(obj).normalize() for obj in processed_target_flows + ] + + source_flows = [ + NormalizedFlow(original=o, normalized=n, current=copy(n)) + for o, n in zip(original_source_flows, normalized_source_flows) + ] + target_flows = [ + NormalizedFlow(original=o, normalized=n, current=copy(n)) + for o, n in zip(original_target_flows, normalized_target_flows) + ] + + flowmap = Flowmap(source_flows, target_flows) + flowmap.generate_matches() + flowmap.print_statistics() + + stem = f"{source.stem}-{target.stem}" + + with open(output_dir / f"{stem}-unmatched-source.json", "w") as fs: + json.dump( + sorted( + [flow.export() for flow in source_flows if not flow.matched], + key=sorting_function, + ), + fs, + indent=True, + ) + + flowmap.to_randonneur( + source_id=source_id, + target_id=target_id, + contributors=contributors, + mapping_source=Flow.randonneur_mapping(), + mapping_target=Flow.randonneur_mapping(), + version=version, + licenses=licenses, + homepage=homepage, + name=name, + path=output_dir / f"{stem}.json", + ) + flowmap.to_glad(output_dir / f"{stem}.xlsx", missing_source=True) + + return flowmap diff --git a/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/just_different.json b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/just_different.json new file mode 100644 index 0000000..ffba958 --- /dev/null +++ b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/just_different.json @@ -0,0 +1,402 @@ +[ + { + "source": { + "name": "Parathion, methyl" + }, + "target": { + "name": "Methyl parathion" + } + }, + { + "source": { + "name": "Thiocyanic acid (-1 ion)" + }, + "target": { + "name": "Thiocyanate" + } + }, + { + "source": { + "name": "Quizalofop ethyl ester" + }, + "target": { + "name": "Quizalofop-ethyl" + } + }, + { + "source": { + "name": "Prothioconazol" + }, + "target": { + "name": "Prothioconazole" + } + }, + { + "source": { + "name": "Pyraclostrobin (prop)" + }, + "target": { + "name": "Pyraclostrobin" + } + }, + { + "source": { + "name": "Monosodium acid methanearsonate" + }, + "target": { + "name": "MSMA" + } + }, + { + "source": { + "name": "Carbamic acid, [(dibutylamino)thio]methyl-, 2,3-dihydro-2,2-dimethyl-7-benzofuranyl ester" + }, + "target": { + "name": "Carbosulfan" + } + }, + { + "source": { + "name": "Benzene, 1-methyl-2-nitro-" + }, + "target": { + "name": "o-Nitrotoluene" + } + }, + { + "source": { + "name": "Alkane (unspecified)" + }, + "target": { + "name": "Hydrocarbons, aliphatic, alkanes, unspecified" + } + }, + { + "source": { + "name": "AOX (Adsorbable Organic Halogens)" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "AOX, Adsorbable Organic Halogen as Cl" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "BOD5 (Biological Oxygen Demand)" + }, + "target": { + "name": "BOD5, Biological Oxygen Demand" + } + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand)" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + } + }, + { + "source": { + "name": "Wood, unspecified, standing/m3" + }, + "target": { + "name": "Wood, unspecified, standing" + } + }, + { + "source": { + "name": "Particulates, < 2.5 um" + }, + "target": { + "name": "Particulate Matter, < 2.5 um" + } + }, + { + "source": { + "name": "Particulates, > 10 um" + }, + "target": { + "name": "Particulate Matter, > 10 um" + } + }, + { + "source": { + "name": "Particulates, > 2.5 um, and < 10um" + }, + "target": { + "name": "Particulate Matter, > 2.5 um and < 10um" + } + }, + { + "source": { + "name": "Sand" + }, + "target": { + "name": "Sand, unspecified" + } + }, + { + "source": { + "name": "Potassium chloride" + }, + "target": { + "name": "Sylvite" + } + }, + { + "source": { + "name": "Sodium tetrahydroborate" + }, + "target": { + "name": "Sodium tetrahydridoborate" + } + }, + { + "source": { + "name": "Toluene, 2-chloro-" + }, + "target": { + "name": "o-Chlorotoluene" + } + }, + { + "source": { + "name": "Pentane, 2,2,4-trimethyl-" + }, + "target": { + "name": "2,2,4-Trimethylpentane" + } + }, + { + "source": { + "name": "Dioxin, 2,3,7,8 Tetrachlorodibenzo-p-" + }, + "target": { + "name": "Dioxins, measured as 2,3,7,8-tetrachlorodibenzo-p-dioxin" + } + }, + { + "source": { + "name": "Discarded fish, demersal" + }, + "target": { + "name": "Discarded fish, demersal, to ocean" + } + }, + { + "source": { + "name": "Methane, tetrachloro-, CFC-10" + }, + "target": { + "name": "Carbon tetrachloride" + } + }, + { + "source": { + "name": "Methane, tetrafluoro-, CFC-14" + }, + "target": { + "name": "Tetrafluoromethane" + } + }, + { + "source": { + "name": "Metolachlor, (S)" + }, + "target": { + "name": "Metolachlor" + } + }, + { + "source": { + "name": "Methane, chlorofluoro-, HCFC-31" + }, + "target": { + "name": "Chloro-fluoromethane" + } + }, + { + "source": { + "name": "Metam-sodium dihydrate" + }, + "target": { + "name": "Metam-sodium" + } + }, + { + "source": { + "name": "Gas, natural, 36 MJ per m3" + }, + "target": { + "name": "Gas, natural, in ground" + } + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining, 36 MJ per m3" + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining" + } + }, + { + "source": { + "name": "Discarded fish, pelagic" + }, + "target": { + "name": "Discarded fish, pelagic, to ocean" + } + }, + { + "source": { + "name": "Dipropylthiocarbamic acid S-ethyl ester" + }, + "target": { + "name": "EPTC" + } + }, + { + "source": { + "name": "Oxydemeton methyl" + }, + "target": { + "name": "Oxydemeton-methyl" + } + }, + { + "source": { + "name": "Thiazole, 2-(thiocyanatemethylthio)benzo-" + }, + "target": { + "name": "TCMTB" + } + }, + { + "source": { + "name": "Tri-allate" + }, + "target": { + "name": "Triallate" + } + }, + { + "source": { + "name": "Cesium (I)" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Cesium" + }, + "target": { + "name": "Caesium" + } + }, + { + "source": { + "name": "Dimethyl formamide" + }, + "target": { + "name": "N,N-Dimethylformamide" + } + }, + { + "source": { + "name": "Methane, land transformation" + }, + "target": { + "name": "Methane, from soil or biomass stock" + } + }, + { + "source": { + "name": "Carbon dioxide, land transformation" + }, + "target": { + "name": "Carbon dioxide, from soil or biomass stock" + } + }, + { + "source": { + "name": "Carbon monoxide, land transformation" + }, + "target": { + "name": "Carbon monoxide, from soil or biomass stock" + } + }, + { + "source": { + "name": "Nitrogen, atmospheric" + }, + "target": { + "name": "Nitrogen" + } + }, + { + "source": { + "name": "Butyric acid, 4-(2,4-dichlorophenoxy)-" + }, + "target": { + "name": "2,4-DB" + } + }, + { + "source": { + "name": "Benzo(a)anthracene" + }, + "target": { + "name": "Benz(a)anthracene" + } + }, + { + "source": { + "name": "Oil, crude, 43.4 MJ per kg" + }, + "target": { + "name": "Oil, crude" + } + }, + { + "source": { + "name": "Argon-40/kg" + }, + "target": { + "name": "Argon" + } + }, + { + "source": { + "name": "1-Butanol" + }, + "target": { + "name": "Butanol" + } + }, + { + "source": { + "name": "Metaldehyde (tetramer)" + }, + "target": { + "name": "Metaldehyde" + } + }, + { + "source": { + "name": "Roundup" + }, + "target": { + "name": "Glyphosate" + } + } +] diff --git a/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/land_use_not_in_ecoinvent.json b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/land_use_not_in_ecoinvent.json new file mode 100644 index 0000000..80cc558 --- /dev/null +++ b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/land_use_not_in_ecoinvent.json @@ -0,0 +1,470 @@ +[ + { + "source": { + "name": "Transformation, from pasture and meadow, organic" + }, + "target": { + "name": "Transformation, from pasture, man made, extensive" + } + }, + { + "source": { + "name": "Transformation, to pasture and meadow, organic" + }, + "target": { + "name": "Transformation, to pasture, man made, extensive" + } + }, + { + "source": { + "name": "Transformation, from arable, organic" + }, + "target": { + "name": "Transformation, from arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to arable, organic" + }, + "target": { + "name": "Transformation, to arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to industrial area, built up" + }, + "target": { + "name": "Transformation, to industrial area", + "context": [ + "natural resource", + "land" + ] + } + }, + { + "source": { + "name": "Transformation, from agriculture" + }, + "target": { + "name": "Transformation, from annual crop" + } + }, + { + "source": { + "name": "Transformation, from annual crop, non-irrigated, fallow" + }, + "target": { + "name": "Transformation, from pasture, man made" + } + }, + { + "source": { + "name": "Transformation, from forest, intensive, clear-cutting" + }, + "target": { + "name": "Transformation, from forest, intensive" + } + }, + { + "source": { + "name": "Transformation, from forest, used" + }, + "target": { + "name": "Transformation, from forest, intensive" + } + }, + { + "source": { + "name": "Transformation, from grassland" + }, + "target": { + "name": "Transformation, from grassland, natural (non-use)" + } + }, + { + "source": { + "name": "Transformation, from grassland/pasture/meadow" + }, + "target": { + "name": "Transformation, from pasture, man made" + } + }, + { + "source": { + "name": "Transformation, from industrial area, benthos" + }, + "target": { + "name": "Transformation, from seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, from industrial area, built up" + }, + "target": { + "name": "Transformation, from industrial area" + } + }, + { + "source": { + "name": "Transformation, from industrial area, vegetation" + }, + "target": { + "name": "Transformation, from industrial area" + } + }, + { + "source": { + "name": "Transformation, from permanent crop, fruit" + }, + "target": { + "name": "Transformation, from permanent crop, irrigated" + } + }, + { + "source": { + "name": "Transformation, from tropical rain forest" + }, + "target": { + "name": "Transformation, from forest, extensive" + } + }, + { + "source": { + "name": "Transformation, from unspecified, used" + }, + "target": { + "name": "Transformation, from unspecified" + } + }, + { + "source": { + "name": "Transformation, to agriculture" + }, + "target": { + "name": "Transformation, to annual crop" + } + }, + { + "source": { + "name": "Transformation, to annual crop, fallow" + }, + "target": { + "name": "Transformation, to arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to annual crop, non-irrigated, fallow" + }, + "target": { + "name": "Transformation, to annual crop, non-irrigated, extensive" + } + }, + { + "source": { + "name": "Transformation, to dump site, benthos" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, clear-cutting" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, normal" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, short-cycle" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, used" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to grassland/pasture/meadow" + }, + "target": { + "name": "Transformation, to pasture, man made" + } + }, + { + "source": { + "name": "Transformation, to industrial area, benthos" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to industrial area, vegetation" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to permanent crop, fruit, intensive" + }, + "target": { + "name": "Transformation, to permanent crop, irrigated" + } + }, + { + "source": { + "name": "Transformation, to sea and ocean" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to traffic area, road embankment" + }, + "target": { + "name": "Transformation, to traffic area, road network" + } + }, + { + "source": { + "name": "Transformation, to unspecified, used" + }, + "target": { + "name": "Transformation, to unspecified" + } + }, + { + "source": { + "name": "Transformation, to urban/industrial fallow" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to water bodies, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Transformation, to water courses, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Transformation, to lakes, artificial" + }, + "target": { + "name": "Transformation, to lake, artificial" + } + }, + { + "source": { + "name": "Transformation, to rivers, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Occupation, lakes, artificial" + }, + "target": { + "name": "Occupation, lake, artificial" + } + }, + { + "source": { + "name": "Occupation, rivers, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, water bodies, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, agriculture" + }, + "target": { + "name": "Occupation, annual crop" + } + }, + { + "source": { + "name": "Occupation, dump site, benthos" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, forest, intensive, normal" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, forest, intensive, short-cycle" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, forest, used" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, grassland/pasture/meadow" + }, + "target": { + "name": "Occupation, pasture, man made" + } + }, + { + "source": { + "name": "Occupation, industrial area, benthos" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, industrial area, built up" + }, + "target": { + "name": "Occupation, industrial area" + } + }, + { + "source": { + "name": "Occupation, industrial area, vegetation" + }, + "target": { + "name": "Occupation, industrial area" + } + }, + { + "source": { + "name": "Occupation, permanent crop, fruit, intensive" + }, + "target": { + "name": "Occupation, permanent crop, irrigated" + } + }, + { + "source": { + "name": "Occupation, sea and ocean" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, sea and ocean" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, traffic area" + }, + "target": { + "name": "Occupation, traffic area, road network" + } + }, + { + "source": { + "name": "Occupation, traffic area, road embankment" + }, + "target": { + "name": "Occupation, traffic area, road network" + } + }, + { + "source": { + "name": "Occupation, unspecified, used" + }, + "target": { + "name": "Occupation, unspecified" + } + }, + { + "source": { + "name": "Occupation, water bodies, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, water courses, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, wetland" + }, + "target": { + "name": "Occupation, inland waterbody, unspecified" + } + } +] diff --git a/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/ores.json b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/ores.json new file mode 100644 index 0000000..80e7508 --- /dev/null +++ b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/ores.json @@ -0,0 +1,203 @@ +[ + { + "source": { + "name": "Bauxite" + }, + "target": { + "name": "Gangue" + } + }, + { + "source": { + "name": "Copper ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 0.38%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Pb 0.014%, in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Coal, 18 MJ per kg" + }, + "target": { + "name": "Coal, hard" + } + }, + { + "source": { + "name": "Coal, brown, 10 MJ per kg" + }, + "target": { + "name": "Coal, brown" + } + }, + { + "source": { + "name": "Coal, brown, 8 MJ per kg" + }, + "target": { + "name": "Coal, brown" + } + }, + { + "source": { + "name": "Crude oil" + }, + "target": { + "name": "Oil, crude" + } + }, + { + "source": { + "name": "Energy, from biomass" + }, + "target": { + "name": "Energy, gross calorific value, in biomass" + } + }, + { + "source": { + "name": "Energy, from coal" + }, + "target": { + "name": "Energy, gross calorific value, in biomass" + } + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining/m3" + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining", + "unit": "Sm3" + } + }, + { + "source": { + "name": "Silver, Ag 9.7E-4%, Au 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Zinc" + } + }, + { + "source": { + "name": "Lead, Pb 0.014%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, in ore" + }, + "target": { + "name": "Lead" + } + }, + { + "source": { + "name": "Nickel, Ni 2.3E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Nickel, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Platinum, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Gold, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Platinum, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Palladium, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Palladium, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Clay" + }, + "target": { + "name": "Clay, bentonite" + } + }, + { + "source": { + "name": "Rhodium, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Rhodium, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Rhodium" + } + } +] diff --git a/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/regionalized_random.json b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/regionalized_random.json new file mode 100644 index 0000000..bdd8d22 --- /dev/null +++ b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/regionalized_random.json @@ -0,0 +1,124 @@ +[ + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, GLO", + "context": [ + "natural resource", + "in water" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, GLO", + "context": [ + "natural resource" + ] + }, + "target": { + "name": "Water, unspecified natural origin", + "context": [ + "natural resource", + "in water" + ] + } + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, GLO", + "context": [ + "natural resource" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Water, unspecified natural origin, CN", + "context": [ + "natural resource", + "land" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + }, + "location": "CN" + }, + { + "source": { + "name": "Water, fresh, CN" + }, + "target": { + "name": "Water, unspecified natural origin", + "context": [ + "natural resource", + "in water" + ] + }, + "location": "CN" + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, Europe without Switzerland", + "context": [ + "natural resource", + "in water" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + }, + "location": "Europe without Switzerland" + }, + { + "source": { + "name": "Water, cooling, unspecified natural origin, CO" + }, + "target": { + "name": "Water, cooling, unspecified natural origin" + }, + "location": "CO" + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand), KR" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + }, + "location": "KR" + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand), US" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + }, + "location": "US" + }, + { + "source": { + "name": "NMVOC, non-methane volatile organic compounds, KR" + }, + "target": { + "name": "NMVOC, non-methane volatile organic compounds, unspecified origin" + }, + "location": "KR" + }, + { + "source": { + "name": "NMVOC, non-methane volatile organic compounds, US" + }, + "target": { + "name": "NMVOC, non-methane volatile organic compounds, unspecified origin" + }, + "location": "US" + } +] diff --git a/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/unit_conversions.json b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/unit_conversions.json new file mode 100644 index 0000000..306cd0f --- /dev/null +++ b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/unit_conversions.json @@ -0,0 +1,108 @@ +[ + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining/m3", + "unit": "cubic meter", + "context": ["natural resource", "in ground"] + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining", + "unit": "Sm3" + }, + "conversion_factor": 1.0 + }, + { + "source": { + "name": "Gas, natural/m3", + "unit": "m3", + "context": ["natural resource", "in ground"] + }, + "target": { + "name": "Gas, natural", + "unit": "Sm3" + }, + "conversion_factor": 1.0 + }, + { + "source": { + "name": "Energy, from peat", + "unit": "MJ" + }, + "target": { + "name": "Peat", + "unit": "kg" + }, + "conversion_factor": 9.9, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from uranium", + "unit": "MJ" + }, + "target": { + "name": "Uranium", + "unit": "kg" + }, + "conversion_factor": 560000.0, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from coal, brown", + "unit": "MJ" + }, + "target": { + "name": "Coal, brown", + "unit": "kg" + }, + "conversion_factor": 9.9, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from gas, natural", + "unit": "MJ" + }, + "target": { + "name": "Gas, natural", + "unit": "Sm3" + }, + "conversion_factor": 40.3, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from oil sand (10% bitumen)", + "unit": "MJ" + }, + "target": { + "name": "Oil, crude", + "unit": "kg" + }, + "conversion_factor": 45.8, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from oil sand (100% bitumen)", + "unit": "MJ" + }, + "target": { + "name": "Oil, crude", + "unit": "kg" + }, + "conversion_factor": 45.8, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from hydro power" + }, + "target": { + "name": "Energy, potential (in hydropower reservoir), converted" + }, + "conversion_factor": 40.3, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + } +] diff --git a/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/water_misc.json b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/water_misc.json new file mode 100644 index 0000000..1c7f6b0 --- /dev/null +++ b/src/flowmapper/manual_matching/data/simapro_ecoinvent_310/water_misc.json @@ -0,0 +1,34 @@ +[ + { + "source": { + "name": "Water/m3" + }, + "target": { + "name": "Water" + } + }, + { + "source": { + "name": "Water, unspecified natural origin/m3" + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Water, cooling, unspecified natural origin/m3" + }, + "target": { + "name": "Water, cooling, unspecified natural origin" + } + }, + { + "source": { + "name": "Resources, rain water, water, withdrawal" + }, + "target": { + "name": "Water, unspecified natural origin" + } + } +] diff --git a/src/flowmapper/manual_matching/results/simapro-2024-ecoinvent-3.10-biosphere.json b/src/flowmapper/manual_matching/results/simapro-2024-ecoinvent-3.10-biosphere.json new file mode 100644 index 0000000..63f9046 --- /dev/null +++ b/src/flowmapper/manual_matching/results/simapro-2024-ecoinvent-3.10-biosphere.json @@ -0,0 +1,3784 @@ +{ + "name": "SimaPro-2024-to-ecoinvent-3.10-elementary-flows", + "description": "Generic manual matching of SimaPro elementary flows to ecoinvent 3.10 elementary flows. Generated by flowmapper.manual_matches.simapro_ecoinvent_310.generate_simapro_ecoinvent_310_manual_matches`.", + "contributors": [ + { + "title": "Chris Mutel", + "path": "https://chris.mutel.org/", + "roles": [ + "author" + ] + } + ], + "created": "2025-07-17T20:43:15.053778+00:00", + "version": "2.0.0", + "licenses": [ + { + "name": "CC-BY-4.0", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "title": "Creative Commons Attribution 4.0 International" + } + ], + "graph_context": [ + "edges" + ], + "mapping": { + "source": { + "expression language": "like JSONPath", + "labels": { + "identifier": "Process[*].\"Process identifier\".text", + "name": "Process[*].Products[*].text[0]", + "platform_id": "Process[*].\"Platform Identifier\"", + "unit": [ + "[\"Emissions to air/\", Process[*].\"Emissions to air\".[2]]", + "[\"Emissions to soil/\", Process[*].\"Emissions to soil\".[2]]", + "[\"Emissions to water/\", Process[*].\"Emissions to water\".[2]]", + "[\"Resources/\", Process[*].\"Resources\".[2]]" + ], + "context": [ + "[\"Emissions to air/\", Process[*].\"Emissions to air\".[1]]", + "[\"Emissions to soil/\", Process[*].\"Emissions to soil\".[1]]", + "[\"Emissions to water/\", Process[*].\"Emissions to water\".[1]]", + "[\"Resources/\", Process[*].\"Resources\".[1]]" + ] + } + }, + "target": { + "expression language": "XPath", + "labels": { + "name": "//*:elementaryExchange/*:name/text()", + "unit": "//*:elementaryExchange/*:unitName/text()", + "uuid": "//*:elementaryExchange/@elementaryExchangeId", + "formula": "//*:elementaryExchange/@formula", + "context": [ + "//*:elementaryExchange/*:compartment/*:compartment/text()", + "//*:elementaryExchange/*:compartment/*:subcompartment/text()" + ] + } + } + }, + "source_id": "SimaPro-2024", + "target_id": "ecoinvent-3.10-biosphere", + "homepage": "https://github.com/cmutel/flowmapper", + "update": [ + { + "source": { + "name": "Caesium", + "context": "air/high. pop." + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "air/indoor" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "air/low. pop." + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "air/low. pop., long-term" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "air/stratosphere + troposphere" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to air" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to air/high. pop." + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to air/low. pop." + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to air/low. pop., long-term" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to air/stratosphere + troposphere" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to soil" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to soil/agricultural" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to soil/forestry" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to soil/industrial" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "water/groundwater" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "water/groundwater, long-term" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "water/fossilwater" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "water/river" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water/groundwater" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water/groundwater, long-term" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water/lake" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water/ocean" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water/river" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Caesium", + "context": "emissions to water/river, long-term" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Calcium", + "context": "air/high. pop." + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "air/indoor" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "air/low. pop." + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "air/low. pop., long-term" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "air/stratosphere + troposphere" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to air" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to air/high. pop." + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to air/low. pop." + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to air/low. pop., long-term" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to air/stratosphere + troposphere" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to soil" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to soil/agricultural" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to soil/forestry" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to soil/industrial" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "water/groundwater" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "water/groundwater, long-term" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "water/fossilwater" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "water/river" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water/groundwater" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water/groundwater, long-term" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water/lake" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water/ocean" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water/river" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Calcium", + "context": "emissions to water/river, long-term" + }, + "target": { + "name": "Calcium II" + } + }, + { + "source": { + "name": "Sodium", + "context": "air/high. pop." + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "air/indoor" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "air/low. pop." + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "air/low. pop., long-term" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "air/stratosphere + troposphere" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to air" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to air/high. pop." + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to air/low. pop." + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to air/low. pop., long-term" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to air/stratosphere + troposphere" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to soil" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to soil/agricultural" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to soil/forestry" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to soil/industrial" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "water/groundwater" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "water/groundwater, long-term" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "water/fossilwater" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "water/river" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water/groundwater" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water/groundwater, long-term" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water/lake" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water/ocean" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water/river" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Sodium", + "context": "emissions to water/river, long-term" + }, + "target": { + "name": "Sodium I" + } + }, + { + "source": { + "name": "Strontium", + "context": "air/high. pop." + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "air/indoor" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "air/low. pop." + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "air/low. pop., long-term" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "air/stratosphere + troposphere" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to air" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to air/high. pop." + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to air/low. pop." + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to air/low. pop., long-term" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to air/stratosphere + troposphere" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to soil" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to soil/agricultural" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to soil/forestry" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to soil/industrial" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "water/groundwater" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "water/groundwater, long-term" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "water/fossilwater" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "water/river" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water/groundwater" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water/groundwater, long-term" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water/lake" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water/ocean" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water/river" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Strontium", + "context": "emissions to water/river, long-term" + }, + "target": { + "name": "Strontium II" + } + }, + { + "source": { + "name": "Parathion, methyl" + }, + "target": { + "name": "Methyl parathion" + } + }, + { + "source": { + "name": "Thiocyanic acid (-1 ion)" + }, + "target": { + "name": "Thiocyanate" + } + }, + { + "source": { + "name": "Quizalofop ethyl ester" + }, + "target": { + "name": "Quizalofop-ethyl" + } + }, + { + "source": { + "name": "Prothioconazol" + }, + "target": { + "name": "Prothioconazole" + } + }, + { + "source": { + "name": "Pyraclostrobin (prop)" + }, + "target": { + "name": "Pyraclostrobin" + } + }, + { + "source": { + "name": "Monosodium acid methanearsonate" + }, + "target": { + "name": "MSMA" + } + }, + { + "source": { + "name": "Carbamic acid, [(dibutylamino)thio]methyl-, 2,3-dihydro-2,2-dimethyl-7-benzofuranyl ester" + }, + "target": { + "name": "Carbosulfan" + } + }, + { + "source": { + "name": "Benzene, 1-methyl-2-nitro-" + }, + "target": { + "name": "o-Nitrotoluene" + } + }, + { + "source": { + "name": "Alkane (unspecified)" + }, + "target": { + "name": "Hydrocarbons, aliphatic, alkanes, unspecified" + } + }, + { + "source": { + "name": "AOX (Adsorbable Organic Halogens)" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "AOX, Adsorbable Organic Halogen as Cl" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "BOD5 (Biological Oxygen Demand)" + }, + "target": { + "name": "BOD5, Biological Oxygen Demand" + } + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand)" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + } + }, + { + "source": { + "name": "Wood, unspecified, standing/m3" + }, + "target": { + "name": "Wood, unspecified, standing" + } + }, + { + "source": { + "name": "Particulates, < 2.5 um" + }, + "target": { + "name": "Particulate Matter, < 2.5 um" + } + }, + { + "source": { + "name": "Particulates, > 10 um" + }, + "target": { + "name": "Particulate Matter, > 10 um" + } + }, + { + "source": { + "name": "Particulates, > 2.5 um, and < 10um" + }, + "target": { + "name": "Particulate Matter, > 2.5 um and < 10um" + } + }, + { + "source": { + "name": "Sand" + }, + "target": { + "name": "Sand, unspecified" + } + }, + { + "source": { + "name": "Potassium chloride" + }, + "target": { + "name": "Sylvite" + } + }, + { + "source": { + "name": "Sodium tetrahydroborate" + }, + "target": { + "name": "Sodium tetrahydridoborate" + } + }, + { + "source": { + "name": "Toluene, 2-chloro-" + }, + "target": { + "name": "o-Chlorotoluene" + } + }, + { + "source": { + "name": "Pentane, 2,2,4-trimethyl-" + }, + "target": { + "name": "2,2,4-Trimethylpentane" + } + }, + { + "source": { + "name": "2,4-D, dimethylamine salt" + }, + "target": { + "name": "2,4-D dimethylamine salt" + } + }, + { + "source": { + "name": "Dioxin, 2,3,7,8 Tetrachlorodibenzo-p-" + }, + "target": { + "name": "Dioxins, measured as 2,3,7,8-tetrachlorodibenzo-p-dioxin" + } + }, + { + "source": { + "name": "Discarded fish, demersal" + }, + "target": { + "name": "Discarded fish, demersal, to ocean" + } + }, + { + "source": { + "name": "Methane, tetrachloro-, CFC-10" + }, + "target": { + "name": "Carbon tetrachloride" + } + }, + { + "source": { + "name": "Methane, tetrafluoro-, CFC-14" + }, + "target": { + "name": "Tetrafluoromethane" + } + }, + { + "source": { + "name": "Metolachlor, (S)" + }, + "target": { + "name": "Metolachlor" + } + }, + { + "source": { + "name": "Methane, chlorofluoro-, HCFC-31" + }, + "target": { + "name": "Chloro-fluoromethane" + } + }, + { + "source": { + "name": "Metam-sodium dihydrate" + }, + "target": { + "name": "Metam-sodium" + } + }, + { + "source": { + "name": "Gas, natural, 36 MJ per m3" + }, + "target": { + "name": "Gas, natural, in ground" + } + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining, 36 MJ per m3" + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining" + } + }, + { + "source": { + "name": "Discarded fish, pelagic" + }, + "target": { + "name": "Discarded fish, pelagic, to ocean" + } + }, + { + "source": { + "name": "Dipropylthiocarbamic acid S-ethyl ester" + }, + "target": { + "name": "EPTC" + } + }, + { + "source": { + "name": "Oxydemeton methyl" + }, + "target": { + "name": "Oxydemeton-methyl" + } + }, + { + "source": { + "name": "Thiazole, 2-(thiocyanatemethylthio)benzo-" + }, + "target": { + "name": "TCMTB" + } + }, + { + "source": { + "name": "Tri-allate" + }, + "target": { + "name": "Triallate" + } + }, + { + "source": { + "name": "Cesium (I)" + }, + "target": { + "name": "Caesium I" + } + }, + { + "source": { + "name": "Dimethyl formamide" + }, + "target": { + "name": "N,N-Dimethylformamide" + } + }, + { + "source": { + "name": "Methane, land transformation" + }, + "target": { + "name": "Methane, from soil or biomass stock" + } + }, + { + "source": { + "name": "Carbon dioxide, land transformation" + }, + "target": { + "name": "Carbon dioxide, from soil or biomass stock" + } + }, + { + "source": { + "name": "Carbon monoxide, land transformation" + }, + "target": { + "name": "Carbon monoxide, from soil or biomass stock" + } + }, + { + "source": { + "name": "Nitrogen, atmospheric" + }, + "target": { + "name": "Nitrogen" + } + }, + { + "source": { + "name": "Butyric acid, 4-(2,4-dichlorophenoxy)-" + }, + "target": { + "name": "2,4-DB" + } + }, + { + "source": { + "name": "Benzo(a)anthracene" + }, + "target": { + "name": "Benz(a)anthracene" + } + }, + { + "source": { + "name": "Oil, crude, 43.4 MJ per kg" + }, + "target": { + "name": "Oil, crude" + } + }, + { + "source": { + "name": "Argon-40/kg" + }, + "target": { + "name": "Argon" + } + }, + { + "source": { + "name": "1-Butanol" + }, + "target": { + "name": "Butanol" + } + }, + { + "source": { + "name": "Metaldehyde (tetramer)" + }, + "target": { + "name": "Metaldehyde" + } + }, + { + "source": { + "name": "Roundup" + }, + "target": { + "name": "Glyphosate", + "context": [ + "air", + "non-urban air or from high stacks" + ] + } + }, + { + "source": { + "name": "Transformation, from pasture and meadow, organic" + }, + "target": { + "name": "Transformation, from pasture, man made, extensive" + } + }, + { + "source": { + "name": "Transformation, to pasture and meadow, organic" + }, + "target": { + "name": "Transformation, to pasture, man made, extensive" + } + }, + { + "source": { + "name": "Transformation, from arable, organic" + }, + "target": { + "name": "Transformation, from arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to arable, organic" + }, + "target": { + "name": "Transformation, to arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to industrial area, built up" + }, + "target": { + "name": "Transformation, to industrial area", + "context": [ + "natural resource", + "land" + ] + } + }, + { + "source": { + "name": "Transformation, from agriculture" + }, + "target": { + "name": "Transformation, from annual crop" + } + }, + { + "source": { + "name": "Transformation, from annual crop, non-irrigated, fallow" + }, + "target": { + "name": "Transformation, from pasture, man made" + } + }, + { + "source": { + "name": "Transformation, from forest, intensive, clear-cutting" + }, + "target": { + "name": "Transformation, from forest, intensive" + } + }, + { + "source": { + "name": "Transformation, from forest, used" + }, + "target": { + "name": "Transformation, from forest, intensive" + } + }, + { + "source": { + "name": "Transformation, from grassland" + }, + "target": { + "name": "Transformation, from grassland, natural (non-use)" + } + }, + { + "source": { + "name": "Transformation, from grassland/pasture/meadow" + }, + "target": { + "name": "Transformation, from pasture, man made" + } + }, + { + "source": { + "name": "Transformation, from industrial area, benthos" + }, + "target": { + "name": "Transformation, from seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, from industrial area, built up" + }, + "target": { + "name": "Transformation, from industrial area" + } + }, + { + "source": { + "name": "Transformation, from industrial area, vegetation" + }, + "target": { + "name": "Transformation, from industrial area" + } + }, + { + "source": { + "name": "Transformation, from permanent crop, fruit" + }, + "target": { + "name": "Transformation, from permanent crop, irrigated" + } + }, + { + "source": { + "name": "Transformation, from tropical rain forest" + }, + "target": { + "name": "Transformation, from forest, extensive" + } + }, + { + "source": { + "name": "Transformation, from unspecified, used" + }, + "target": { + "name": "Transformation, from unspecified" + } + }, + { + "source": { + "name": "Transformation, to agriculture" + }, + "target": { + "name": "Transformation, to annual crop" + } + }, + { + "source": { + "name": "Transformation, to annual crop, fallow" + }, + "target": { + "name": "Transformation, to arable land, unspecified use" + } + }, + { + "source": { + "name": "Transformation, to annual crop, non-irrigated, fallow" + }, + "target": { + "name": "Transformation, to annual crop, non-irrigated, extensive" + } + }, + { + "source": { + "name": "Transformation, to dump site, benthos" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, clear-cutting" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, normal" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, intensive, short-cycle" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to forest, used" + }, + "target": { + "name": "Transformation, to forest, intensive" + } + }, + { + "source": { + "name": "Transformation, to grassland/pasture/meadow" + }, + "target": { + "name": "Transformation, to pasture, man made" + } + }, + { + "source": { + "name": "Transformation, to industrial area, benthos" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to industrial area, built up" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to industrial area, vegetation" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to permanent crop, fruit, intensive" + }, + "target": { + "name": "Transformation, to permanent crop, irrigated" + } + }, + { + "source": { + "name": "Transformation, to sea and ocean" + }, + "target": { + "name": "Transformation, to seabed, unspecified" + } + }, + { + "source": { + "name": "Transformation, to traffic area, road embankment" + }, + "target": { + "name": "Transformation, to traffic area, road network" + } + }, + { + "source": { + "name": "Transformation, to unspecified, used" + }, + "target": { + "name": "Transformation, to unspecified" + } + }, + { + "source": { + "name": "Transformation, to urban/industrial fallow" + }, + "target": { + "name": "Transformation, to industrial area" + } + }, + { + "source": { + "name": "Transformation, to water bodies, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Transformation, to water courses, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Transformation, to lakes, artificial" + }, + "target": { + "name": "Transformation, to lake, artificial" + } + }, + { + "source": { + "name": "Transformation, to rivers, artificial" + }, + "target": { + "name": "Transformation, to river, artificial" + } + }, + { + "source": { + "name": "Occupation, lakes, artificial" + }, + "target": { + "name": "Occupation, lake, artificial" + } + }, + { + "source": { + "name": "Occupation, rivers, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, water bodies, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, agriculture" + }, + "target": { + "name": "Occupation, annual crop" + } + }, + { + "source": { + "name": "Occupation, dump site, benthos" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, forest, intensive, normal" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, forest, intensive, short-cycle" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, forest, used" + }, + "target": { + "name": "Occupation, forest, intensive" + } + }, + { + "source": { + "name": "Occupation, grassland/pasture/meadow" + }, + "target": { + "name": "Occupation, pasture, man made" + } + }, + { + "source": { + "name": "Occupation, industrial area, benthos" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, industrial area, built up" + }, + "target": { + "name": "Occupation, industrial area" + } + }, + { + "source": { + "name": "Occupation, industrial area, vegetation" + }, + "target": { + "name": "Occupation, industrial area" + } + }, + { + "source": { + "name": "Occupation, permanent crop, fruit, intensive" + }, + "target": { + "name": "Occupation, permanent crop, irrigated" + } + }, + { + "source": { + "name": "Occupation, sea and ocean" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, sea and ocean" + }, + "target": { + "name": "Occupation, seabed, unspecified" + } + }, + { + "source": { + "name": "Occupation, traffic area" + }, + "target": { + "name": "Occupation, traffic area, road network" + } + }, + { + "source": { + "name": "Occupation, traffic area, road embankment" + }, + "target": { + "name": "Occupation, traffic area, road network" + } + }, + { + "source": { + "name": "Occupation, unspecified, used" + }, + "target": { + "name": "Occupation, unspecified" + } + }, + { + "source": { + "name": "Occupation, water bodies, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, water courses, artificial" + }, + "target": { + "name": "Occupation, river, artificial" + } + }, + { + "source": { + "name": "Occupation, wetland" + }, + "target": { + "name": "Occupation, inland waterbody, unspecified" + } + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, GLO", + "context": [ + "natural resource", + "in water" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, GLO", + "context": [ + "natural resource" + ] + }, + "target": { + "name": "Water, unspecified natural origin", + "context": [ + "natural resource", + "in water" + ] + } + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, GLO", + "context": [ + "natural resource" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Water, unspecified natural origin, CN", + "context": [ + "natural resource", + "land" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + }, + "location": "CN" + }, + { + "source": { + "name": "Water, fresh, CN" + }, + "target": { + "name": "Water, unspecified natural origin", + "context": [ + "natural resource", + "in water" + ] + }, + "location": "CN" + }, + { + "source": { + "name": "Water (ersatz), net cons., non-irrigation, Europe without Switzerland", + "context": [ + "natural resource", + "in water" + ] + }, + "target": { + "name": "Water, unspecified natural origin" + }, + "location": "Europe without Switzerland" + }, + { + "source": { + "name": "Water, cooling, unspecified natural origin, CO" + }, + "target": { + "name": "Water, cooling, unspecified natural origin" + }, + "location": "CO" + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand), KR" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + }, + "location": "KR" + }, + { + "source": { + "name": "COD (Chemical Oxygen Demand), US" + }, + "target": { + "name": "COD, Chemical Oxygen Demand" + }, + "location": "US" + }, + { + "source": { + "name": "NMVOC, non-methane volatile organic compounds, KR" + }, + "target": { + "name": "NMVOC, non-methane volatile organic compounds, unspecified origin" + }, + "location": "KR" + }, + { + "source": { + "name": "NMVOC, non-methane volatile organic compounds, US" + }, + "target": { + "name": "NMVOC, non-methane volatile organic compounds, unspecified origin" + }, + "location": "US" + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining/m3", + "unit": "cubic meter", + "context": [ + "natural resource", + "in ground" + ] + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining", + "unit": "Sm3" + }, + "conversion_factor": 1.0 + }, + { + "source": { + "name": "Gas, natural/m3", + "unit": "m3", + "context": [ + "natural resource", + "in ground" + ] + }, + "target": { + "name": "Gas, natural", + "unit": "Sm3" + }, + "conversion_factor": 1.0 + }, + { + "source": { + "name": "Energy, from peat", + "unit": "MJ" + }, + "target": { + "name": "Peat", + "unit": "kg" + }, + "conversion_factor": 9.9, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from uranium", + "unit": "MJ" + }, + "target": { + "name": "Uranium", + "unit": "kg" + }, + "conversion_factor": 560000.0, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from coal, brown", + "unit": "MJ" + }, + "target": { + "name": "Coal, brown", + "unit": "kg" + }, + "conversion_factor": 9.9, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from gas, natural", + "unit": "MJ" + }, + "target": { + "name": "Gas, natural", + "unit": "Sm3" + }, + "conversion_factor": 40.3, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from oil sand (10% bitumen)", + "unit": "MJ" + }, + "target": { + "name": "Oil, crude", + "unit": "kg" + }, + "conversion_factor": 45.8, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from oil sand (100% bitumen)", + "unit": "MJ" + }, + "target": { + "name": "Oil, crude", + "unit": "kg" + }, + "conversion_factor": 45.8, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Energy, from hydro power" + }, + "target": { + "name": "Energy, potential (in hydropower reservoir), converted" + }, + "conversion_factor": 40.3, + "comment": "Conversion factor from ecoinvent 3.10 CED LCIA factors" + }, + { + "source": { + "name": "Water/m3" + }, + "target": { + "name": "Water" + } + }, + { + "source": { + "name": "Water, unspecified natural origin/m3" + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Water, cooling, unspecified natural origin/m3" + }, + "target": { + "name": "Water, cooling, unspecified natural origin" + } + }, + { + "source": { + "name": "Resources, rain water, water, withdrawal" + }, + "target": { + "name": "Water, unspecified natural origin" + } + }, + { + "source": { + "name": "Barite" + }, + "target": { + "name": "Barium" + } + }, + { + "source": { + "name": "Bauxite" + }, + "target": { + "name": "Gangue" + } + }, + { + "source": { + "name": "Copper ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 0.38%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Pb 0.014%, in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Copper, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Coal, 18 MJ per kg" + }, + "target": { + "name": "Coal, hard" + } + }, + { + "source": { + "name": "Coal, brown, 10 MJ per kg" + }, + "target": { + "name": "Coal, brown" + } + }, + { + "source": { + "name": "Coal, brown, 8 MJ per kg" + }, + "target": { + "name": "Coal, brown" + } + }, + { + "source": { + "name": "Crude oil" + }, + "target": { + "name": "Oil, crude" + } + }, + { + "source": { + "name": "Energy, from biomass" + }, + "target": { + "name": "Energy, gross calorific value, in biomass" + } + }, + { + "source": { + "name": "Energy, from coal" + }, + "target": { + "name": "Energy, gross calorific value, in biomass" + } + }, + { + "source": { + "name": "Gas, mine, off-gas, process, coal mining/m3" + }, + "target": { + "name": "Gas, mine, off-gas, process, coal mining", + "unit": "Sm3" + } + }, + { + "source": { + "name": "Silver, Ag 9.7E-4%, Au 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Zinc" + } + }, + { + "source": { + "name": "Lead, Pb 0.014%, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, in ore" + }, + "target": { + "name": "Lead" + } + }, + { + "source": { + "name": "Nickel, Ni 2.3E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Nickel, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Platinum, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Gold, Au 9.7E-4%, Ag 9.7E-4%, Zn 0.63%, Cu 0.38%, Pb 0.014%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Platinum, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Palladium, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Palladium, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Clay" + }, + "target": { + "name": "Clay, bentonite" + } + }, + { + "source": { + "name": "Rhodium, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Rhodium, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Palladium, Pd 1.6E-6%, in mixed ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Phenol, 2,4-dichloro" + }, + "target": { + "name": "2,4-Dichlorophenol" + } + }, + { + "source": { + "name": "Gold, Au 6.8E-4%, Ag 1.5E-4%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Pd, Pd 2.0E-4%, Pt 4.8E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Indium, 0.005% in sulfide, In 0.003%, Pb, Zn, Ag, Cd" + }, + "target": { + "name": "Indium" + } + }, + { + "source": { + "name": "Hydrogen-3, Tritium" + }, + "target": { + "name": "Tritium" + } + }, + { + "source": { + "name": "Silver, Ag 4.2E-3%, Au 1.1E-4%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Methane, trifluoro-, HFC-23" + }, + "target": { + "name": "Trifluoromethane" + } + }, + { + "source": { + "name": "Phthalate, butyl-benzyl-" + }, + "target": { + "name": "Butyl benzyl phthalate" + } + }, + { + "source": { + "name": "Methane, tetrachloro-, R-10" + }, + "target": { + "name": "Carbon tetrachloride" + } + }, + { + "source": { + "name": "AOX, Adsorbable Organic Halogen" + }, + "target": { + "name": "AOX, Adsorbable Organic Halides" + } + }, + { + "source": { + "name": "Benzene, pentachloro-" + }, + "target": { + "name": "Pentachlorobenzene" + } + }, + { + "source": { + "name": "Xylene" + }, + "target": { + "name": "Xylenes, unspecified" + } + }, + { + "source": { + "name": "Zinc, Zn 3.1%, in mixed ore" + }, + "target": { + "name": "Zinc" + } + }, + { + "source": { + "name": "Molybdenum, 0.014% in sulfide, Mo 8.2E-3% and Cu 0.81% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Silver, 3.2ppm in sulfide, Ag 1.2ppm, Cu and Te, in crude ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Methane, dichloro-, HCC-30" + }, + "target": { + "name": "Dichloromethane" + } + }, + { + "source": { + "name": "Silver, Ag 1.5E-5%, Au 5.4E-4%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Gold, Au 9.7E-4%, in mixed ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Prohexadione-calcium" + }, + "target": { + "name": "Prohexadione calcium" + } + }, + { + "source": { + "name": "Pd, Pd 7.3E-4%, Pt 2.5E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Palladium" + } + }, + { + "source": { + "name": "Gallium, 0.014% in bauxite" + }, + "target": { + "name": "Gallium" + } + }, + { + "source": { + "name": "Samarium, 0.3% in bastnasite, 0.03% in crude ore" + }, + "target": { + "name": "Samarium" + } + }, + { + "source": { + "name": "Chloroethylene" + }, + "target": { + "name": "Vinyl chloride" + } + }, + { + "source": { + "name": "Gold, Au 1.3E-4%, Ag 4.6E-5%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Barite, 15% in crude ore" + }, + "target": { + "name": "Barium" + } + }, + { + "source": { + "name": "Hexamethyldisilizane" + }, + "target": { + "name": "Hexamethyldisilazane" + } + }, + { + "source": { + "name": "Phthalate, dimethyl-" + }, + "target": { + "name": "Dimethyl phthalate" + } + }, + { + "source": { + "name": "Lanthanum, 7.2% in bastnasite, 0.72% in crude ore" + }, + "target": { + "name": "Lanthanum" + } + }, + { + "source": { + "name": "Ethane, 1,1,2-trichloro-1,2,2-trifluoro-, CFC-113" + }, + "target": { + "name": "1,1,2-Trichloro-1,2,2-trifluoroethane" + } + }, + { + "source": { + "name": "Benzene, chloro-" + }, + "target": { + "name": "Monochlorobenzene" + } + }, + { + "source": { + "name": "Chromium, 25.5% in chromite, 11.6% in crude ore" + }, + "target": { + "name": "Chromium" + } + }, + { + "source": { + "name": "Europium, 0.06% in bastnasite, 0.006% in crude ore" + }, + "target": { + "name": "Europium" + } + }, + { + "source": { + "name": "Fluorine, 4.5% in apatite, 1% in crude ore" + }, + "target": { + "name": "Fluorine" + } + }, + { + "source": { + "name": "Pt, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Silver, Ag 1.5E-4%, Au 6.8E-4%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Fluorine, 4.5% in apatite, 3% in crude ore" + }, + "target": { + "name": "Fluorine" + } + }, + { + "source": { + "name": "Copper, Cu 6.8E-1%, in mixed ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Fosetyl-aluminium" + }, + "target": { + "name": "Fosetyl-Al" + } + }, + { + "source": { + "name": "Silver, Ag 1.8E-6%, in mixed ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Gold, Au 1.0E-7%, in mixed ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Cu, Cu 3.2E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "1,3-Dioxolan-2-one" + }, + "target": { + "name": "Ethylene carbonate" + } + }, + { + "source": { + "name": "Ethyne" + }, + "target": { + "name": "Acetylene" + } + }, + { + "source": { + "name": "Cyclohexane (for all cycloalkanes)" + }, + "target": { + "name": "Hydrocarbons, aliphatic, alkanes, cyclic" + } + }, + { + "source": { + "name": "Methane, tetrafluoro-, R-14" + }, + "target": { + "name": "Tetrafluoromethane" + } + }, + { + "source": { + "name": "2,4-D amines" + }, + "target": { + "name": "2,4-D dimethylamine salt" + } + }, + { + "source": { + "name": "Aluminium, 24% in bauxite, 11% in crude ore" + }, + "target": { + "name": "Aluminium" + } + }, + { + "source": { + "name": "1,2-Dichlorobenzene" + }, + "target": { + "name": "o-Dichlorobenzene" + } + }, + { + "source": { + "name": "Pt, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Ethane, 1,2-dichloro-1,1,2,2-tetrafluoro-, CFC-114" + }, + "target": { + "name": "1,2-Dichloro-1,1,2,2-tetrafluoroethane" + } + }, + { + "source": { + "name": "Sodium sulphate, various forms" + }, + "target": { + "name": "Sodium sulfate" + } + }, + { + "source": { + "name": "Dimethenamid" + }, + "target": { + "name": "Dimethenamide" + } + }, + { + "source": { + "name": "Ethane, 1,1-dichloro-1-fluoro-, HCFC-141b" + }, + "target": { + "name": "1,1-Dichloro-1-fluoroethane" + } + }, + { + "source": { + "name": "Propylamine" + }, + "target": { + "name": "Propyl amine" + } + }, + { + "source": { + "name": "2-Propanol" + }, + "target": { + "name": "Isopropanol" + } + }, + { + "source": { + "name": "Zinc, 9.0% in sulfide, Zn 5.3%, Pb, Ag, Cd, In" + }, + "target": { + "name": "Zinc" + } + }, + { + "source": { + "name": "Cu, Cu 5.2E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Ni 3.7E-2% in ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Gold, Au 1.4E-4%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Copper, 0.99% in sulfide, Cu 0.36% and Mo 8.2E-3% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Iron, 46% in ore, 25% in crude ore" + }, + "target": { + "name": "Iron" + } + }, + { + "source": { + "name": "Magnesium, 0.13% in water" + }, + "target": { + "name": "Magnesium" + } + }, + { + "source": { + "name": "Ni, Ni 3.7E-2%, Pt 4.8E-4%, Pd 2.0E-4%, Rh 2.4E-5%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Copper, 2.19% in sulfide, Cu 1.83% and Mo 8.2E-3% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Phosphorus, 18% in apatite, 4% in crude ore" + }, + "target": { + "name": "Phosphorus" + } + }, + { + "source": { + "name": "Methane, monochloro-, R-40" + }, + "target": { + "name": "Methylchloride" + } + }, + { + "source": { + "name": "Argon-40" + }, + "target": { + "name": "Argon" + } + }, + { + "source": { + "name": "Chlortoluron" + }, + "target": { + "name": "Chlorotoluron" + } + }, + { + "source": { + "name": "Methyl borate" + }, + "target": { + "name": "Trimethyl borate" + } + }, + { + "source": { + "name": "Rhodium, Rh 1.6E-7%, in mixed ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Gold, Au 4.9E-5%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Gold, Au 4.3E-4%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Copper, 1.18% in sulfide, Cu 0.39% and Mo 8.2E-3% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "2-Amino-3-chloro-1,4-naphthoquinone" + }, + "target": { + "name": "Quinoclamine" + } + }, + { + "source": { + "name": "Ethane, pentafluoro-, HFC-125" + }, + "target": { + "name": "Pentafluoroethane" + } + }, + { + "source": { + "name": "Molybdenum, 0.022% in sulfide, Mo 8.2E-3% and Cu 0.36% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Ni, Ni 2.3E+0%, Pt 2.5E-4%, Pd 7.3E-4%, Rh 2.0E-5%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Chlorosilane, trimethyl-" + }, + "target": { + "name": "Trimethylchlorosilane" + } + }, + { + "source": { + "name": "Benzene, ethyl-" + }, + "target": { + "name": "Ethyl benzene" + } + }, + { + "source": { + "name": "Acetic acid, trifluoro-" + }, + "target": { + "name": "Trifluoroacetic acid" + } + }, + { + "source": { + "name": "Copper, 0.97% in sulfide, Cu 0.36% and Mo 4.1E-2% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Cobalt, Co 5.0E-2%, in mixed ore" + }, + "target": { + "name": "Cobalt" + } + }, + { + "source": { + "name": "Ethane, 1,1,1-trifluoro-, HFC-143a" + }, + "target": { + "name": "1,1,1-Trifluoroethane" + } + }, + { + "source": { + "name": "Methane, difluoro-, HFC-32" + }, + "target": { + "name": "Difluoromethane" + } + }, + { + "source": { + "name": "Methane, trichlorofluoro-, CFC-11" + }, + "target": { + "name": "Trichlorofluoromethane" + } + }, + { + "source": { + "name": "Phthalate, dibutyl-" + }, + "target": { + "name": "Dibutyl phthalate" + } + }, + { + "source": { + "name": "2-chlorobenzaldehyde" + }, + "target": { + "name": "o-Chlorobenzaldehyde" + } + }, + { + "source": { + "name": "t-Butylamine" + }, + "target": { + "name": "t-Butyl amine" + } + }, + { + "source": { + "name": "Monophenyltin" + }, + "target": { + "name": "Phenyltin" + } + }, + { + "source": { + "name": "Gold, Au 5.4E-4%, Ag 1.5E-5%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Copper, 1.25% in sulfide, Cu 0.24% and Zn 0,1% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Ethane, hexachloro-" + }, + "target": { + "name": "Hexachloroethane" + } + }, + { + "source": { + "name": "Methane, chlorodifluoro-, HCFC-22" + }, + "target": { + "name": "Chlorodifluoromethane" + } + }, + { + "source": { + "name": "Iron, 72% in magnetite, 14% in crude ore" + }, + "target": { + "name": "Iron" + } + }, + { + "source": { + "name": "Rhenium, in crude ore" + }, + "target": { + "name": "Rhenium" + } + }, + { + "source": { + "name": "Silver, Ag 9.7E-4%, in mixed ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Helium, 0.08% in natural gas" + }, + "target": { + "name": "Helium, in natural gas" + } + }, + { + "source": { + "name": "Iodine, 0.03% in water" + }, + "target": { + "name": "Iodine" + } + }, + { + "source": { + "name": "Molybdenum, 0.025% in sulfide, Mo 8.2E-3% and Cu 0.39% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Platinum, Pt 4.7E-7%, in mixed ore" + }, + "target": { + "name": "Platinum" + } + }, + { + "source": { + "name": "Phenol, pentachloro-" + }, + "target": { + "name": "Pentachlorophenol" + } + }, + { + "source": { + "name": "Chromium IV" + }, + "target": { + "name": "Chromium VI" + } + }, + { + "source": { + "name": "Ethane, 2-chloro-1,1,1,2-tetrafluoro-, HCFC-124" + }, + "target": { + "name": "2-Chloro-1,1,1,2-tetrafluoroethane" + } + }, + { + "source": { + "name": "Molybdenum, 0.016% in sulfide, Mo 8.2E-3% and Cu 0.27% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Methane, chlorotrifluoro-, CFC-13" + }, + "target": { + "name": "Chlorotrifluoromethane" + } + }, + { + "source": { + "name": "Molybdenum, 0.11% in sulfide, Mo 4.1E-2% and Cu 0.36% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Methane, dichlorofluoro-, HCFC-21" + }, + "target": { + "name": "Dichlorofluoromethane" + } + }, + { + "source": { + "name": "Ethane, 2,2-dichloro-1,1,1-trifluoro-, HCFC-123" + }, + "target": { + "name": "2,2-Dichloro-1,1,1-trifluoroethane" + } + }, + { + "source": { + "name": "Gold, Au 1.8E-4%, in mixed ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Molybdenum, 0.022% in sulfide, Mo 8.2E-3% and Cu 0.22% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Zirconium, 50% in zircon, 0.39% in crude ore" + }, + "target": { + "name": "Zirconium" + } + }, + { + "source": { + "name": "Copper, 0.52% in sulfide, Cu 0.27% and Mo 8.2E-3% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Gold, Au 1.1E-4%, Ag 4.2E-3%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Lead, Pb 3.6E-1%, in mixed ore" + }, + "target": { + "name": "Lead" + } + }, + { + "source": { + "name": "Silver, Ag 4.6E-5%, Au 1.3E-4%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Propene" + }, + "target": { + "name": "Propylene" + } + }, + { + "source": { + "name": "Magnesite, 60% in crude ore" + }, + "target": { + "name": "Magnesite" + } + }, + { + "source": { + "name": "Silver, Ag 2.1E-4%, Au 2.1E-4%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Gadolinium, 0.15% in bastnasite, 0.015% in crude ore" + }, + "target": { + "name": "Gadolinium" + } + }, + { + "source": { + "name": "Neodymium, 4% in bastnasite, 0.4% in crude ore" + }, + "target": { + "name": "Neodymium" + } + }, + { + "source": { + "name": "Phthalate, dioctyl-" + }, + "target": { + "name": "Dioctyl phthalate" + } + }, + { + "source": { + "name": "Bromine, 0.23% in water" + }, + "target": { + "name": "Bromine" + } + }, + { + "source": { + "name": "Copper, 1.42% in sulfide, Cu 0.81% and Mo 8.2E-3% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "2-Methylpentane" + }, + "target": { + "name": "Isohexane" + } + }, + { + "source": { + "name": "Fresh water (obsolete)" + }, + "target": { + "name": "Water" + } + }, + { + "source": { + "name": "Coal, hard, unspecified" + }, + "target": { + "name": "Coal, hard" + } + }, + { + "source": { + "name": "2,2,4-Trimethyl pentane" + }, + "target": { + "name": "2,2,4-Trimethylpentane" + } + }, + { + "source": { + "name": "Fluorspar, 92%" + }, + "target": { + "name": "Fluorspar" + } + }, + { + "source": { + "name": "Lithium, 0.15% in brine" + }, + "target": { + "name": "Lithium" + } + }, + { + "source": { + "name": "Rh, Rh 2.4E-5%, Pt 4.8E-4%, Pd 2.0E-4%, Ni 3.7E-2%, Cu 5.2E-2% in ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Methane, chloro-fluoro-, HCFC-31" + }, + "target": { + "name": "Chloro-fluoromethane" + } + }, + { + "source": { + "name": "Manganese, 35.7% in sedimentary deposit, 14.2% in crude ore" + }, + "target": { + "name": "Manganese" + } + }, + { + "source": { + "name": "Ethane, 1,1,1-trichloro-, HCFC-140" + }, + "target": { + "name": "1,1,1-Trichloroethane" + } + }, + { + "source": { + "name": "Elemental carbon" + }, + "target": { + "name": "Carbon" + } + }, + { + "source": { + "name": "TiO2, 54% in ilmenite, 18% in crude ore" + }, + "target": { + "name": "Titanium" + } + }, + { + "source": { + "name": "Phosphorus, 18% in apatite, 12% in crude ore" + }, + "target": { + "name": "Phosphorus" + } + }, + { + "source": { + "name": "Dipropylamine" + }, + "target": { + "name": "Dipropyl amine" + } + }, + { + "source": { + "name": "Nickel, 1.13% in sulfide, Ni 0.76% and Cu 0.76% in crude ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Silver, 0.007% in sulfide, Ag 0.004%, Pb, Zn, Cd, In" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Cyhalothrin, gamma-" + }, + "target": { + "name": "Gamma-cyhalothrin" + } + }, + { + "source": { + "name": "Praseodymium, 0.42% in bastnasite, 0.042% in crude ore" + }, + "target": { + "name": "Praseodymium" + } + }, + { + "source": { + "name": "Rh, Rh 2.0E-5%, Pt 2.5E-4%, Pd 7.3E-4%, Ni 2.3E+0%, Cu 3.2E+0% in ore" + }, + "target": { + "name": "Rhodium" + } + }, + { + "source": { + "name": "Molybdenum, 0.010% in sulfide, Mo 8.2E-3% and Cu 1.83% in crude ore" + }, + "target": { + "name": "Molybdenum" + } + }, + { + "source": { + "name": "Gold, Au 2.1E-4%, Ag 2.1E-4%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "2-Methyl-1-propanol" + }, + "target": { + "name": "Isobutanol" + } + }, + { + "source": { + "name": "Benzene, hexachloro-" + }, + "target": { + "name": "Hexachlorobenzene" + } + }, + { + "source": { + "name": "Butylcarbamate, iodopropynyl" + }, + "target": { + "name": "Iodopropynyl butylcarbamate" + } + }, + { + "source": { + "name": "Tantalum, 81.9% in tantalite, 1.6E-4% in crude ore" + }, + "target": { + "name": "Tantalum" + } + }, + { + "source": { + "name": "Bicyclorpyrone" + }, + "target": { + "name": "Bicyclopyrone" + } + }, + { + "source": { + "name": "TiO2, 54% in ilmenite, 2.6% in crude ore" + }, + "target": { + "name": "Titanium" + } + }, + { + "source": { + "name": "Tin, 79% in cassiterite, 0.1% in crude ore" + }, + "target": { + "name": "Tin" + } + }, + { + "source": { + "name": "Copper, 0.59% in sulfide, Cu 0.22% and Mo 8.2E-3% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Cadmium, 0.30% in sulfide, Cd 0.18%, Pb, Zn, Ag, In" + }, + "target": { + "name": "Cadmium" + } + }, + { + "source": { + "name": "Pyrethrins" + }, + "target": { + "name": "Pyrethrum" + } + }, + { + "source": { + "name": "Copper, Cu 0.2%, in mixed ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Sylvite, 25 % in sylvinite" + }, + "target": { + "name": "Sylvite" + } + }, + { + "source": { + "name": "Silver, 0.01% in crude ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Toluene, 2-chloro" + }, + "target": { + "name": "o-Chlorotoluene" + } + }, + { + "source": { + "name": "Ethane, 1,1-difluoro-, HFC-152a" + }, + "target": { + "name": "1,1-Difluoroethane" + } + }, + { + "source": { + "name": "Nickel, Ni 2.5E+0%, in mixed ore" + }, + "target": { + "name": "Nickel" + } + }, + { + "source": { + "name": "Ethane, chloropentafluoro-, CFC-115" + }, + "target": { + "name": "Monochloropentafluoroethane" + } + }, + { + "source": { + "name": "Silver, Ag 5.4E-3%, in mixed ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Zinc, Zn 0.63%, in mixed ore" + }, + "target": { + "name": "Zinc" + } + }, + { + "source": { + "name": "Methane, dichlorodifluoro-, CFC-12" + }, + "target": { + "name": "Dichlorodifluoromethane" + } + }, + { + "source": { + "name": "Silver, Ag 7.6E-5%, Au 9.7E-5%, in ore" + }, + "target": { + "name": "Silver" + } + }, + { + "source": { + "name": "Kieserite, 25% in crude ore" + }, + "target": { + "name": "Kieserite" + } + }, + { + "source": { + "name": "Kaolinite, 24% in crude ore" + }, + "target": { + "name": "Kaolinite" + } + }, + { + "source": { + "name": "Lead, Pb 0.014%, in mixed ore" + }, + "target": { + "name": "Lead" + } + }, + { + "source": { + "name": "Copper, Cu 0.38%, in mixed ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "TiO2, 95% in rutile, 0.40% in crude ore" + }, + "target": { + "name": "Titanium" + } + }, + { + "source": { + "name": "Copper, 1.13% in sulfide, Cu 0.76% and Ni 0.76% in crude ore" + }, + "target": { + "name": "Copper" + } + }, + { + "source": { + "name": "Ethane, 1,2-dichloro-" + }, + "target": { + "name": "Ethylene dichloride" + } + }, + { + "source": { + "name": "Ethane, 1-chloro-1,1-difluoro-, HCFC-142b" + }, + "target": { + "name": "1-Chloro-1,1-difluoroethane" + } + }, + { + "source": { + "name": "Cerium, 24% in bastnasite, 2.4% in crude ore" + }, + "target": { + "name": "Cerium" + } + }, + { + "source": { + "name": "Gold, Au 7.1E-4%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Ethane, 1,1,1,2-tetrafluoro-, HFC-134a" + }, + "target": { + "name": "1,1,1,2-Tetrafluoroethane" + } + }, + { + "source": { + "name": "Gold, Au 9.7E-5%, Ag 7.6E-5%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Ethane, hexafluoro-, HFC-116" + }, + "target": { + "name": "Hexafluoroethane" + } + }, + { + "source": { + "name": "Lead, 5.0% in sulfide, Pb 3.0%, Zn, Ag, Cd, In" + }, + "target": { + "name": "Lead" + } + }, + { + "source": { + "name": "Gold, Au 6.7E-4%, in ore" + }, + "target": { + "name": "Gold" + } + }, + { + "source": { + "name": "Ethane, 1,1,2-trichloro-" + }, + "target": { + "name": "1,1,2-Trichloroethane" + } + }, + { + "source": { + "name": "Tellurium, 0.5ppm in sulfide, Te 0.2ppm, Cu and Ag, in crude ore" + }, + "target": { + "name": "Tellurium" + } + }, + { + "source": { + "name": "Nickel, 1.98% in silicates, 1.04% in crude ore" + }, + "target": { + "name": "Nickel" + } + } + ] +} \ No newline at end of file diff --git a/src/flowmapper/manual_matching/simapro_ecoinvent_310.py b/src/flowmapper/manual_matching/simapro_ecoinvent_310.py new file mode 100644 index 0000000..7c760f8 --- /dev/null +++ b/src/flowmapper/manual_matching/simapro_ecoinvent_310.py @@ -0,0 +1,85 @@ +import json +from pathlib import Path + +import randonneur as rn +import randonneur_data as rd + +data_dir = Path(__file__).parent / "data" / "simapro_ecoinvent_310" +results_dir = Path(__file__).parent / "results" +base_data_dir = Path(__file__).parent.parent / "data" + + +def generate_simapro_ecoinvent_310_manual_matches( + output_dir: Path = results_dir, + filename: str = "simapro-2024-ecoinvent-3.10-biosphere.json", + add_to_randonneur_data: bool = False, +) -> Path: + # Manually add some ecoinvent 3.9 -> 3.10 conversions which don't apply to resources + mapping_files = [ + "just_different.json", + "land_use_not_in_ecoinvent.json", + "regionalized_random.json", + "unit_conversions.json", + "water_misc.json", + "ores.json", + ] + non_resources = { + "Caesium": "Caesium I", + "Calcium": "Calcium II", + "Sodium": "Sodium I", + "Strontium": "Strontium II", + } + non_resource_categories = [ + obj["source"]["context"] + for obj in json.load( + open(base_data_dir / "simapro-2023-ecoinvent-3-contexts.json") + )["update"] + if obj["target"]["context"][0] != "natural resource" + ] + + data = [ + {"source": {"name": key, "context": context}, "target": {"name": value}} + for key, value in non_resources.items() + for context in non_resource_categories + ] + for fn in mapping_files: + data.extend(json.load(open(data_dir / fn))) + + registry = rd.Registry() + migration = registry.get_file("ecoinvent-3.9.1-biosphere-ecoinvent-3.10-biosphere") + name_change = { + (pair["source"]["name"], pair["target"]["name"]) + for pair in migration["replace"] + if "name" in pair["target"] + and "name" in pair["source"] + and pair["source"]["name"] != pair["target"]["name"] + and pair["source"]["name"] not in non_resources + } + assert len(name_change) == len({a for a, b in name_change}) + data.extend( + [{"source": {"name": a}, "target": {"name": b}} for a, b in name_change] + ) + + dp = rn.Datapackage( + name="SimaPro-2024-to-ecoinvent-3.10-elementary-flows", + description="Generic manual matching of SimaPro elementary flows to ecoinvent 3.10 elementary flows. Generated by flowmapper.manual_matches.simapro_ecoinvent_310.generate_simapro_ecoinvent_310_manual_matches`.", + contributors=[ + { + "title": "Chris Mutel", + "path": "https://chris.mutel.org/", + "roles": ["author"], + } + ], + mapping_source=rn.MappingConstants.SIMAPRO_CSV, + mapping_target=rn.MappingConstants.ECOSPOLD2_BIO, + source_id="SimaPro-2024", + target_id="ecoinvent-3.10-biosphere", + homepage="https://github.com/cmutel/flowmapper", + version="2.0.0", + ) + dp.add_data("update", data) + dp.to_json(output_dir / filename) + + +if __name__ == "__main__": + generate_simapro_ecoinvent_310_manual_matches() diff --git a/src/flowmapper/match.py b/src/flowmapper/match.py new file mode 100644 index 0000000..09aea02 --- /dev/null +++ b/src/flowmapper/match.py @@ -0,0 +1,573 @@ +import itertools +import logging +from collections.abc import Callable +from functools import partial + +from rapidfuzz.distance.DamerauLevenshtein import distance + +from flowmapper.domain import Flow, Match, MatchCondition, NormalizedFlow +from flowmapper.utils import FlowTransformationContext, apply_randonneur, toolz + +logger = logging.getLogger(__name__) + +# Note: It might seem like running these functions in parallel would be much faster, but in +# practice it doesn't seem to be. The memory overhead of copying over very large sets of target +# flows means parallel execution was twice as slow, at least in my testing. + + +def get_matches( + source_flows: list[NormalizedFlow], + target_flows: list[NormalizedFlow], + comment: str, + function_name: str, + match_condition: MatchCondition, + conversion_factors: list[float] | None = None, +) -> list[Match]: + if not target_flows: + return [] + + matches = [] + + # Providing conversion_factors only makes sense if there is a single target flow + # Otherwise you have M-to-N problem + if conversion_factors is None: + cfs = itertools.repeat(None) + else: + if not len(conversion_factors) == len(source_flows): + raise ValueError( + f"`conversion_factors` (length {len(conversion_factors)}) must have same length as `source_flows` (length {len(source_flows)})" + ) + cfs = conversion_factors + + for conversion_factor, source in zip(cfs, source_flows): + targets = [flow for flow in target_flows if flow.unit_compatible(flow)] + if len(targets) > 1: + # Try find most-appropriate match if more than one is present. Added because ecoinvent + # deprecated most stratospheric emissions and redirected them to air, unspecified, so + # now all air, unspecified emissions have multiple targets. + targets = [ + target + for target in targets + if target.normalized.context == source.normalized.context + ] + if len(targets) == 1: + target = target_flows[0] + source.matched = True + if conversion_factor is None: + conversion_factor = source.conversion_factor(target) + matches.append( + Match( + source=source.original, + target=target.original, + function_name=function_name, + comment=comment or "", + condition=match_condition, + conversion_factor=conversion_factor, + ) + ) + + return matches + + +def match_identical_identifier( + source_flows: list[NormalizedFlow], + target_flows: list[NormalizedFlow], +) -> list[Match]: + matches = [] + + for source_id, sources in toolz.itertoolz.groupby( + lambda x: x.identifier, source_flows + ).items(): + if not source_id: + continue + matches.extend( + get_matches( + source_flows=sources, + # Filter target flows with matching identifier. We don't need to worry about + # duplicate identifiers as `get_matches` will only allow a single result target + target_flows=[ + flow for flow in target_flows if source_id == flow.identifier + ], + comment=f"Shared target-unique identifier: {source_id}", + function_name="match_identical_identifier", + match_condition=MatchCondition.exact, + ) + ) + + return matches + + +def match_identical_cas_numbers( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (cas_number, context, location), sources in toolz.itertoolz.groupby( + lambda x: (x.cas_number, x.context, x.location), source_flows + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + flow + for flow in target_flows + if flow.cas_number == cas_number + and flow.context == context + and flow.location == location + ], + comment=f"Shared CAS code with identical context and location: {cas_number}", + function_name="match_identical_cas_numbers", + match_condition=MatchCondition.exact, + ) + ) + + return matches + + +def match_identical_names( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (name, context, oxidation_state, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.context, x.oxidation_state, x.location), source_flows + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + target + for target in target_flows + if target.name == name + and target.context == context + and target.oxidation_state == oxidation_state + and target.location == location + ], + comment=f"Shared normalized name with identical context, oxidation state, and location: {name}", + function_name="match_identical_names", + match_condition=MatchCondition.exact, + ) + ) + + return matches + + +def match_close_names( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (name, context, oxidation_state, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.context, x.oxidation_state, x.location), source_flows + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + target + for target in target_flows + if distance( + str(target.name), str(name), processor=lambda x: x.lower() + ) + < 3 + and target.context == context + and target.oxidation_state == oxidation_state + and target.location == location + ], + comment=f"Name has Damerau Levenshtein edit distance of 2 or lower with identical context, oxidation state, and location: {name}", + function_name="match_close_names", + match_condition=MatchCondition.related, + ) + ) + + return matches + + +def match_ecoinvent_transitive_matching( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + func: Callable[[list[NormalizedFlow]], list[NormalizedFlow]] = partial( + apply_randonneur, + datapackage="ecoinvent-2.2-biosphere-ecoinvent-3.12-biosphere-transitive", + fields=["name", "context"], + ) + + with ( + FlowTransformationContext(source_flows, func) as sf, + FlowTransformationContext(target_flows, func) as tf, + ): + for (name, context, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.context, x.location), sf + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + target + for target in tf + if target.name.lower() == name.lower() + and target.context == context + and target.location == location + ], + comment=f"Shared normalized name when transitively harmonized to ecoinvent 3.12 with identical context and location: {name}", + function_name="match_ecoinvent_transitive_matching", + match_condition=MatchCondition.close, + ) + ) + + return matches + + +def match_with_transformation( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow], transformation: str, fields: list[str] +) -> list[Match]: + matches = [] + + func: Callable[[list[NormalizedFlow]], list[NormalizedFlow]] = partial( + apply_randonneur, + datapackage=transformation, + fields=fields, + ) + + with FlowTransformationContext(source_flows, func) as sf: + for (name, context, oxidation_state, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.context, x.oxidation_state, x.location), sf + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + target + for target in target_flows + if target.name == name + and target.context == context + and target.oxidation_state == oxidation_state + and target.location == location + ], + comment=f"Shared normalized attributes after applying transformation: {transformation}", + function_name="match_with_transformation", + match_condition=MatchCondition.related, + ) + ) + + return matches + + +def match_identical_names_lowercase( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (name, context, oxidation_state, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.context, x.oxidation_state, x.location), source_flows + ).items(): + name = name.lower() + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + flow + for flow in target_flows + if flow.name.lower() == name + and flow.context == context + and flow.oxidation_state == oxidation_state + and flow.location == location + ], + comment=f"Shared normalized lowercase name with identical context, oxidation state, and location: {name}", + function_name="match_identical_names_lowercase", + match_condition=MatchCondition.close, + ) + ) + + return matches + + +def match_identical_names_without_commas( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (name, context, oxidation_state, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.context, x.oxidation_state, x.location), source_flows + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + flow + for flow in target_flows + if flow.name.replace(",", "") == name.replace(",", "") + and flow.context == context + and flow.oxidation_state == oxidation_state + and flow.location == location + ], + comment=f"Shared normalized name with commas removed and identical context, oxidation state, and location: {name}", + match_condition=MatchCondition.close, + function_name="match_identical_names_without_commas", + ) + ) + + return matches + + +def match_resources_with_wrong_subcontext( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (name, oxidation_state, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.oxidation_state, x.location), + filter(lambda f: f.normalized.context.is_resource(), source_flows), + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + flow + for flow in target_flows + if flow.name == name + and flow.normalized.context.is_resource() + and flow.oxidation_state == oxidation_state + and flow.location == location + ], + comment=f"Shared normalized name and resource-type context, with identical oxidation state and location: {name}", + match_condition=MatchCondition.close, + function_name="match_resources_with_wrong_subcontext", + ) + ) + + return matches + + +def match_identical_names_except_missing_suffix( + source_flows: list[Flow], + target_flows: list[Flow], + suffix: str, + comment: str = "Identical names except missing suffix", +) -> dict: + if ( + (f"{s.name.normalized}, {suffix}" == t.name) + or (f"{t.name.normalized}, {suffix}" == s.name) + or (f"{s.name.normalized} {suffix}" == t.name) + or (f"{t.name.normalized} {suffix}" == s.name) + ) and s.context == t.context: + return {"comment": comment} + + +# def match_names_with_roman_numerals_in_parentheses( +# source_flows: list[Flow], target_flows: list[Flow], comment="With/without roman numerals in parentheses" +# ): +# if ( +# rm_parentheses_roman_numerals(s.name.normalized) +# == rm_parentheses_roman_numerals(t.name.normalized) +# and s.context == t.context +# ): +# return {"comment": comment} + + +# def match_custom_names_with_location_codes( +# source_flows: list[Flow], target_flows: list[Flow], comment="Custom names with location code" +# ): +# """Matching which pulls out location codes but also allows for custom name transformations.""" +# match = ends_with_location.search(s.name.normalized) +# if match: +# location = location_reverser[match.group("code")] +# # Don't use replace, it will find e.g. ", fr" in "transformation, from" +# name = s.name.normalized[: -len(match.group())] +# try: +# mapped_name = names_and_locations[name]["target"] +# except KeyError: +# return +# if mapped_name == t.name.normalized and s.context == t.context: +# result = {"comment": comment, "location": location} | names_and_locations[ +# name +# ].get("extra", {}) +# if ( +# s.name.normalized.startswith("water") +# and s.unit.normalized == "cubic_meter" +# and t.unit.normalized == "kilogram" +# ): +# result["conversion_factor"] = 1000 +# elif ( +# s.name.normalized.startswith("water") +# and t.unit.normalized == "cubic_meter" +# and s.unit.normalized == "kilogram" +# ): +# result["conversion_factor"] = 0.001 +# return result + + +# def match_names_with_location_codes( +# source_flows: list[Flow], target_flows: list[Flow], comment="Name matching with location code" +# ): +# match = ends_with_location.search(s.name.normalized) +# if match: +# location = location_reverser[match.group("code")] +# name = s.name.normalized.replace(match.group(), "") +# if name == t.name.normalized and s.context == t.context: +# result = {"comment": comment, "location": location} +# if ( +# s.name.normalized.startswith("water") +# and s.unit.normalized == "cubic_meter" +# and t.unit.normalized == "kilogram" +# ): +# result["conversion_factor"] = 1000.0 +# elif ( +# s.name.normalized.startswith("water") +# and t.unit.normalized == "cubic_meter" +# and s.unit.normalized == "kilogram" +# ): +# result["conversion_factor"] = 0.001 +# return result + + +def match_name_and_parent_context( + source_flows: list[NormalizedFlow], target_flows: list[NormalizedFlow] +) -> list[Match]: + matches = [] + + for (name, oxidation_state, context, location), sources in toolz.itertoolz.groupby( + lambda x: (x.name, x.oxidation_state, x.context, x.location), + filter(lambda f: len(f.context) > 1, source_flows), + ).items(): + matches.extend( + get_matches( + source_flows=sources, + target_flows=[ + flow + for flow in target_flows + if flow.name == name + and flow.context == context[:-1] + and flow.oxidation_state == oxidation_state + and flow.location == location + ], + comment="Shared normalized name and parent context, with identical oxidation state and location", + match_condition=MatchCondition.related, + function_name="match_name_and_parent_context", + ) + ) + + return matches + + +# def match_non_ionic_state( +# source_flows: list[Flow], target_flows: list[Flow], comment="Non-ionic state if no better match" +# ): +# if ( +# (rm_roman_numerals_ionic_state(s.name.normalized) == t.name) +# or (rm_roman_numerals_ionic_state(s.name.normalized) + ", ion" == t.name) +# ) and s.context == t.context: +# return {"comment": comment} + + +def match_biogenic_to_non_fossil( + source_flows: list[Flow], + target_flows: list[Flow], + comment="Biogenic to non-fossil if no better match", +): + if ( + s.name.normalized.removesuffix(", biogenic") + == t.name.normalized.removesuffix(", non-fossil") + and s.context == t.context + ): + return {"comment": comment} + + +def match_resources_with_suffix_in_ground( + source_flows: list[Flow], target_flows: list[Flow] +): + return match_identical_names_except_missing_suffix( + s, + t, + all_source_flows, + all_target_flows, + suffix="in ground", + comment="Resources with suffix in ground", + ) + + +def match_flows_with_suffix_unspecified_origin( + source_flows: list[Flow], target_flows: list[Flow] +): + return match_identical_names_except_missing_suffix( + s, + t, + all_source_flows, + all_target_flows, + suffix="unspecified origin", + comment="Flows with suffix unspecified origin", + ) + + +def match_resources_with_suffix_in_water( + source_flows: list[Flow], target_flows: list[Flow] +): + return match_identical_names_except_missing_suffix( + s, + t, + all_source_flows, + all_target_flows, + suffix="in water", + comment="Resources with suffix in water", + ) + + +def match_resources_with_suffix_in_air( + source_flows: list[Flow], target_flows: list[Flow] +): + return match_identical_names_except_missing_suffix( + s, + t, + all_source_flows, + all_target_flows, + suffix="in air", + comment="Resources with suffix in air", + ) + + +def match_emissions_with_suffix_ion(source_flows: list[Flow], target_flows: list[Flow]): + return match_identical_names_except_missing_suffix( + s, + t, + all_source_flows, + all_target_flows, + suffix="ion", + comment="Match emissions with suffix ion", + ) + + +def match_rules(): + simple_ecoinvent = partial( + match_with_transformation, + transformation="ecoinvent-3.10-biosphere-simapro-2024-biosphere", + fields=["name"], + ) + simple_ecoinvent.__name__ = "match_with_transformation" + + return [ + match_identical_identifier, + match_identical_names, + # match_identical_names_lowercase, + match_identical_names_without_commas, + match_ecoinvent_transitive_matching, + # match_resources_with_suffix_in_ground, + # match_resources_with_suffix_in_water, + # match_resources_with_suffix_in_air, + # match_flows_with_suffix_unspecified_origin, + match_resources_with_wrong_subcontext, + match_name_and_parent_context, + match_close_names, + simple_ecoinvent, + # match_emissions_with_suffix_ion, + # match_names_with_roman_numerals_in_parentheses, + # match_names_with_location_codes, + # match_resource_names_with_location_codes_and_parent_context, + # match_custom_names_with_location_codes, + match_identical_cas_numbers, + # match_non_ionic_state, + # match_biogenic_to_non_fossil, + # match_identical_names_in_preferred_synonyms, + # match_identical_names_in_synonyms, + ] diff --git a/src/flowmapper/oxidation_state.py b/src/flowmapper/oxidation_state.py new file mode 100644 index 0000000..f181b7d --- /dev/null +++ b/src/flowmapper/oxidation_state.py @@ -0,0 +1,59 @@ +import re +from typing import Any, Self + +import roman + +roman_numberals_optional_parentheses = re.compile( + r"[\,\s]+\(?\s*(?P[IVX]+)\s*(?P[+-]*)\)?\s*$", + flags=re.IGNORECASE, +) +numbers_optional_parentheses = re.compile( + r"[\,\s]+\(?\s*(?P[+-]+)(?P[0-9]+)\)?\s*$" +) + + +class OxidationState: + def __init__(self, value: int): + self.value = value + + def __eq__(self, other: Any) -> bool: + if isinstance(other, OxidationState): + return self.value == other.value + else: + return self.value == other + + def __hash__(self) -> int: + return hash(self.value) + + @staticmethod + def has_oxidation_state(obj: str) -> bool: + return roman_numberals_optional_parentheses.search( + obj + ) or numbers_optional_parentheses.search(obj) + + @classmethod + def from_string(cls, obj: str) -> tuple[Self, str]: + if match := roman_numberals_optional_parentheses.search(obj): + obj_dict = match.groupdict() + try: + value = roman.fromRoman(obj_dict["numeral"].upper()) + except roman.InvalidRomanNumeralError: + raise ValueError( + f"{obj_dict['numeral']} in string {obj} is not a valid roman numeral" + ) + if "-" in obj_dict["sign"]: + value *= -1 + elif match := numbers_optional_parentheses.search(obj): + obj_dict = match.groupdict() + value = eval(obj_dict["numeral"].lstrip("0")) + if "-" in obj_dict["sign"]: + value *= -1 + else: + raise ValueError("No match found") + + if value < -5 or value > 9: + raise ValueError( + f"Oxidation state {value} from name {obj} is outside physical bounds of [-5, +9]" + ) + + return OxidationState(value), obj[: match.start()] diff --git a/src/flowmapper/preferred_synonyms.py b/src/flowmapper/preferred_synonyms.py new file mode 100644 index 0000000..cff0348 --- /dev/null +++ b/src/flowmapper/preferred_synonyms.py @@ -0,0 +1,84 @@ +import re + +from flowmapper.domain import Flow + +ROMAN_NUMERAL_PATTERN = re.compile(r"\b\(?[ivx]+[\+-]?\)?\s*$", flags=re.IGNORECASE) +PARENTHESES_PATTERN = re.compile(r"\([1-9]+[\+-]?\)\s*$") + + +def has_roman_numeral_at_end(text: str) -> bool: + """ + Check if a string ends with a roman numeral. + + Args: + text (str): The string to check + + Returns: + bool: True if the string ends with a roman numeral, False otherwise + + """ + return bool(ROMAN_NUMERAL_PATTERN.search(text)) + + +def has_number_pattern_at_end(text: str) -> bool: + """ + Check if a string ends with a pattern like "(2+)". + + Args: + text (str): The string to check + + Returns: + bool: True if the string ends with the number pattern, False otherwise + + """ + return bool(PARENTHESES_PATTERN.search(text)) + + +def match_identical_names_in_preferred_synonyms( + source_flows: list[Flow], + target_flows: list[Flow], + comment: str = "Identical preferred synonyms", +): + if t.synonyms and s.name in t.synonyms and s.context == t.context: + if s.name.normalized in t.name.normalized and ( + has_roman_numeral_at_end(t.name.normalized) + or has_number_pattern_at_end(t.name.normalized) + ): + # Check if there's another target flow with a different name that shares the same synonym + for other_target in all_target_flows: + if ( + other_target is not t + and other_target.name.normalized != t.name.normalized + and other_target.synonyms + and s.name in other_target.synonyms + and other_target.context == s.context + ): + return None + return {"comment": comment} + elif s.synonyms and t.name in s.synonyms and s.context == t.context: + if t.name.normalized in s.name.normalized and ( + has_roman_numeral_at_end(s.name.normalized) + or has_number_pattern_at_end(s.name.normalized) + ): + # Check if there's another target flow with a different name that shares the same synonym + for other_target in all_target_flows: + if ( + other_target is not t + and other_target.name.normalized != t.name.normalized + and other_target.synonyms + and t.name in other_target.synonyms + and other_target.context == s.context + ): + return None + return {"comment": comment} + + +def match_identical_names_in_synonyms( + source_flows: list[Flow], + target_flows: list[Flow], + comment: str = "Identical synonyms", +): + if (t.synonyms and s.name in t.synonyms and s.context == t.context) or ( + s.synonyms and t.name in s.synonyms and s.context == t.context + ): + return {"comment": comment} diff --git a/src/flowmapper/string_field.py b/src/flowmapper/string_field.py new file mode 100644 index 0000000..33e975f --- /dev/null +++ b/src/flowmapper/string_field.py @@ -0,0 +1,23 @@ +from collections import UserString +from typing import Any, Self + +from flowmapper.utils import normalize_str + + +class StringField(UserString): + def normalize(self, lowercase: bool = True) -> Self: + value = normalize_str(self.data) + if lowercase: + value = value.lower() + return type(self)(value) + + def __eq__(self, other: Any) -> bool: + if not self.data: + # Empty strings aren't equal for our use case + return False + elif isinstance(other, StringField): + return self.data == other.data + elif isinstance(other, str): + return self.data == other or self.data == normalize_str(other) + else: + return False diff --git a/src/flowmapper/transformation_mapping.py b/src/flowmapper/transformation_mapping.py new file mode 100644 index 0000000..9b0c1a9 --- /dev/null +++ b/src/flowmapper/transformation_mapping.py @@ -0,0 +1,50 @@ +from collections import UserDict +from functools import partial +from typing import Any, List + +from flowmapper.context import ContextField +from flowmapper.string_field import StringField +from flowmapper.unit import UnitField + +ATTRIBUTE_MAPPING = { + "unit": partial(UnitField, use_lowercase=True), + "context": ContextField, + "identifier": partial(StringField, use_lowercase=True), +} + + +class ComparableFlowMapping(UserDict): + def __init__(self, initialdata: dict): + self.data = { + key: ATTRIBUTE_MAPPING.get(key, StringField)(value) + for key, value in initialdata.items() + } + + def __setitem__(self, key: Any, value: Any) -> None: + self.data[key] = ATTRIBUTE_MAPPING.get(key, StringField)(value) + + def __eq__(self, other: Any) -> bool: + return all(value == other.get(key) for key, value in self.data.items() if value) + + +def prepare_transformations(transformations: List[dict] | None) -> List[dict]: + if not transformations: + return [] + + prepared_transformations = [] + + for transformation_dataset in transformations: + for transformation_mapping in transformation_dataset.get("update", []): + transformation_mapping["source"] = ComparableFlowMapping( + transformation_mapping["source"] + ) + for other_dataset in prepared_transformations: + for other_mapping in other_dataset.get("update", []): + if other_mapping["source"] == transformation_mapping["source"]: + for key, value in other_mapping["target"].items(): + transformation_mapping["source"][key] = value + break + + prepared_transformations.append(transformation_dataset) + + return prepared_transformations diff --git a/src/flowmapper/unit.py b/src/flowmapper/unit.py new file mode 100644 index 0000000..95e4746 --- /dev/null +++ b/src/flowmapper/unit.py @@ -0,0 +1,66 @@ +import importlib.resources as resource +import json +import math +from collections import UserString +from pathlib import Path +from typing import Any, Self + +from pint import UnitRegistry, errors + +from flowmapper.utils import normalize_str + +ureg = UnitRegistry() + +with resource.as_file(resource.files("flowmapper") / "data" / "units.txt") as filepath: + ureg.load_definitions(filepath) + +with open(Path(__file__).parent / "data" / "standard-units-harmonization.json") as f: + UNIT_MAPPING = { + line["source"]["unit"]: line["target"]["unit"] + for line in json.load(f)["update"] + } + + +class UnitField(UserString): + def normalize(self) -> Self: + """Normalize string to fit into our `pint` definitions""" + label = normalize_str(self.data) + if label in UNIT_MAPPING: + label = UNIT_MAPPING[label] + try: + ureg(label) + except errors.UndefinedUnitError: + raise ValueError( + f"Unit {label} is unknown; add to flowmapper `units.txt` or define a mapping in `unit-mapping.json`" + ) + # Makes type checkers happy, if inelegant... + return type(self)(label) + + def is_uri(self, value: str) -> bool: + # Placeholder for when we support glossary entries + return False + + def resolve_uri(self, uri: str) -> None: + # Placeholder + pass + + def __eq__(self, other: Any) -> bool: + if isinstance(other, UnitField): + return self.data == other.data or self.conversion_factor(other) == 1 + else: + return self.data == other + + def compatible(self, other: Any) -> bool: + return math.isfinite(self.conversion_factor(other)) + + def conversion_factor(self, to: Any) -> float: + if not isinstance(to, (UnitField, str)): + result = float("nan") + elif isinstance(to, UnitField) and self.data == to.data: + result = 1.0 + else: + try: + result = ureg(self.data).to(ureg(str(to))).magnitude + except (errors.DimensionalityError, errors.UndefinedUnitError): + result = float("nan") + return result diff --git a/src/flowmapper/utils.py b/src/flowmapper/utils.py new file mode 100644 index 0000000..c5a32cf --- /dev/null +++ b/src/flowmapper/utils.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +import copy +import importlib.resources as resource +import json +import re +import unicodedata +from collections.abc import Callable, Collection, Mapping +from contextlib import AbstractContextManager +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import structlog +from randonneur import Datapackage, MigrationConfig, migrate_nodes +from randonneur_data import Registry + +if TYPE_CHECKING: + from flowmapper.domain import Flow, NormalizedFlow + +logger = structlog.get_logger("flowmapper") +default_registry = Registry() +RESULTS_DIR = Path(__file__).parent / "manual_matching" / "results" + + +with resource.as_file( + resource.files("flowmapper") / "data" / "names_and_locations.json" +) as filepath: + names_and_locations = {o["source"]: o for o in json.load(open(filepath))} + +try: + import cytoolz as toolz +except ImportError: + logger.info("Install `cytoolz` to get a speed up in matching functions") + import toolz + +assert toolz # Do not delete the import call stupid linter + + +def tupleize_context(obj: dict) -> dict: + """Convert `context` value to `tuple` if possible. + + Handles both individual migration objects and full datapackage structures. + For datapackages, iterates through verb keys (like "update", "create") and + processes all migration objects in those lists. + """ + # Handle datapackage structure with verb keys (update, create, etc.) + if isinstance(obj, dict): + # Check if this looks like a datapackage (has verb keys with lists) + verb_keys = ["update", "create", "delete", "rename"] + has_verb_keys = any( + key in obj and isinstance(obj[key], list) for key in verb_keys + ) + + if has_verb_keys: + # This is a datapackage - process each verb's list + for verb in verb_keys: + if verb in obj and isinstance(obj[verb], list): + for migration_obj in obj[verb]: + if isinstance(migration_obj, dict): + tupleize_context(migration_obj) + return obj + + # Handle individual migration object or dict with context + if isinstance(obj, dict): + # Process top-level context if present + if "context" in obj and not isinstance(obj["context"], str): + obj["context"] = as_normalized_tuple(obj["context"]) + + # Recursively process source and target + if isinstance(obj.get("source"), dict): + tupleize_context(obj["source"]) + if isinstance(obj.get("target"), dict): + tupleize_context(obj["target"]) + + return obj + + +MISSING_VALUES = { + "", + "(unknown)", + "(unspecified)", + "null", + "unknown", + "unspecified", +} + + +def as_normalized_tuple(value: Any) -> tuple[str]: + """Convert context inputs to normalized tuple form.""" + if isinstance(value, (tuple, list)): + intermediate = value + elif isinstance(value, str) and "/" in value: + intermediate = list(value.split("/")) + elif isinstance(value, str): + intermediate = [value] + else: + raise ValueError(f"Can't understand input context {value}") + + intermediate = [elem.lower().strip() for elem in intermediate] + + while intermediate and intermediate[-1] in MISSING_VALUES: + if len(intermediate) == 1: + break + intermediate = intermediate[:-1] + + return tuple(intermediate) + + +def load_standard_transformations() -> list: + # with resource.as_file( + # resource.files("flowmapper") / "data" / "standard-units-harmonization.json" + # ) as filepath: + # units = json.load(open(filepath)) + with resource.as_file( + resource.files("flowmapper") / "data" / "simapro-2023-ecoinvent-3-contexts.json" + ) as filepath: + contexts = json.load(open(filepath)) + # return [units, contexts] + return [contexts] + + +def read_migration_files(*filepaths: str | Path) -> list[dict]: + """ + Read and aggregate migration data from multiple JSON files. + + This function opens and reads a series of JSON files, each containing migration data as a list of dicts without the change type. + It aggregates all changes into a single list and returns it wrapped in a dictionary + under the change type 'update'. + + Parameters + ---------- + *filepaths : Path + Variable length argument list of Path objects. + + Returns + ------- + dict + A dictionary containing a single key 'update', which maps to a list. This list is + an aggregation of the data from all the JSON files read. + """ + migration_data = [] + + for filepath in filepaths: + if (RESULTS_DIR / filepath).is_file(): + filepath = RESULTS_DIR / filepath + with open(Path(filepath)) as fs: + migration_data.append(json.load(fs)) + + return migration_data + + +def normalize_str(s: Any) -> str: + if s is not None: + return unicodedata.normalize("NFC", s).strip() + else: + return "" + + +def transform_flow(flow, transformation): + result = copy.copy(flow) + result.update(transformation["target"]) + return result + + +def matcher(source, target): + return all(target.get(key) == value for key, value in source.items()) + + +def rowercase(obj: Any) -> Any: + """Recursively transform everything to lower case recursively""" + if isinstance(obj, str): + return obj.lower() + elif isinstance(obj, Mapping): + return type(obj)([(rowercase(k), rowercase(v)) for k, v in obj.items()]) + elif isinstance(obj, Collection): + return type(obj)([rowercase(o) for o in obj]) + else: + return obj + + +def apply_transformations(obj: dict, transformations: list[dict] | None) -> dict: + if not transformations: + return obj + obj = copy.deepcopy(obj) + lower = rowercase(obj) + + for dataset in transformations: + for transformation_obj in dataset.get("create", []): + if matcher( + transformation_obj, + lower if dataset.get("case-insensitive") else obj, + ): + # Marked an needs to be created; missing in target list + obj["__missing__"] = True + break + for transformation_obj in dataset.get("update", []): + source_to_match = lower if dataset.get("case-insensitive") else obj + if dataset.get("case-insensitive"): + source_transformation = ( + rowercase(transformation_obj["source"]) + if isinstance(transformation_obj["source"], dict) + else transformation_obj["source"] + ) + else: + source_transformation = transformation_obj["source"] + if matcher(source_transformation, source_to_match): + obj.update(transformation_obj["target"]) + if "conversion_factor" in transformation_obj: + obj["conversion_factor"] = transformation_obj["conversion_factor"] + break + + return obj + + +unit_slash = re.compile(r"/(?Pm3|kg)(\,?\s+|\s+|$)") + + +def remove_unit_slash(obj: Flow) -> str: + name = obj.name.data + if match := unit_slash.search(name): + obj_dict = match.groupdict() + if match.end() == len(name): + name = name[: match.start()] + else: + name = name[: match.start()] + ", " + name[match.end() :] + if not obj.unit.compatible(obj_dict["unit"]): + logger.warning( + f"Flow {obj} has unit '{obj.unit}' but name refers to incompatible unit '{obj_dict['unit']}'" + ) + return name + + +def randonneur_as_function( + datapackage: str | Datapackage | dict, + fields: list[str] | None = None, + registry: Registry | None = None, + verbs: list[str] | None = None, +) -> Callable: + """Take a prepared transformation in""" + if registry is None: + registry = default_registry + if verbs is None: + verbs = ["update"] + + if isinstance(datapackage, Datapackage): + datapackage = datapackage.data + elif isinstance(datapackage, str): + datapackage = registry.get_file(datapackage) + elif "update" not in datapackage: + raise KeyError + + return partial( + migrate_nodes, + migrations=tupleize_context(datapackage), + config=MigrationConfig( + verbs=verbs, + case_sensitive=( + False + if "case-insensitive" not in datapackage + else not datapackage.get("case-insensitive") + ), + fields=fields, + ), + ) + + +def apply_randonneur( + flows: list[NormalizedFlow], + datapackage: str | Datapackage | dict, + fields: list[str] | None = None, + registry: Registry | None = None, +) -> list[NormalizedFlow]: + from flowmapper.domain import Flow + + func = randonneur_as_function( + datapackage=datapackage, fields=fields, registry=registry + ) + transformed_data = func(graph=[nf.normalized.to_dict() for nf in flows]) + + for flow, data_dict in zip(flows, transformed_data): + flow.current = Flow.from_dict(data_dict) + + return flows + + +class FlowTransformationContext(AbstractContextManager): + """ + Context manager that applies a function to NormalizedFlows on entry and resets them on exit. + + This context manager is useful when you need to temporarily modify flows for matching + or processing, and want to ensure they are reset to their normalized state afterward. + + Parameters + ---------- + flows : list[NormalizedFlow] + List of NormalizedFlow objects to transform and reset. + function : Callable[[list[NormalizedFlow]], list[NormalizedFlow]] | None + Function to apply to the flows on context entry. The function should take + a list of NormalizedFlow objects and return the modified list. If None, + no transformation is applied. + + Examples + -------- + >>> flows = [NormalizedFlow(...), NormalizedFlow(...)] + >>> def update_func(flows): + ... for flow in flows: + ... flow.update_current(name="Modified") + ... return flows + >>> with FlowTransformationContext(flows, update_func) as modified_flows: + ... # modified_flows contains the transformed flows + ... do_something_with(modified_flows) + >>> # flows are automatically reset to normalized state + """ + + def __init__( + self, + flows: list[NormalizedFlow], + function: Callable[[list[NormalizedFlow]], list[NormalizedFlow]] | None = None, + ): + self.flows = flows + self.function = function + + def __enter__(self) -> list[NormalizedFlow]: + """Apply the function to the flows on entry.""" + if self.function is not None: + self.flows = self.function(self.flows) + return self.flows + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Reset all flows to their normalized state on exit.""" + for flow in self.flows: + flow.reset_current() diff --git a/tests/__snapshots__/test_cli.ambr b/tests/__snapshots__/test_cli.ambr index 3f498d2..a369b76 100644 --- a/tests/__snapshots__/test_cli.ambr +++ b/tests/__snapshots__/test_cli.ambr @@ -2,97 +2,24 @@ # name: test_matched_flows list([ dict({ - 'CAS': '000110-63-4', - 'categories': list([ + 'CAS number': '110-63-4', + 'context': list([ 'Air', '(unspecified)', ]), + 'identifier': None, 'name': '1,4-Butanediol', 'unit': 'kg', }), dict({ - 'categories': list([ + 'CAS number': '', + 'context': list([ 'Emissions to air', 'low. pop.', ]), + 'identifier': None, 'name': 'Ammonia, FR', 'unit': 'kg', }), ]) # --- -# name: test_matched_flows_with_multiple_randonneur_transformations - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, as N', - 'unit': 'kg', - }), - ]) -# --- -# name: test_matched_flows_with_randonneur_transformations - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, as N', - 'unit': 'kg', - }), - ]) -# --- diff --git a/tests/__snapshots__/test_flowmap.ambr b/tests/__snapshots__/test_flowmap.ambr deleted file mode 100644 index 5e2a26c..0000000 --- a/tests/__snapshots__/test_flowmap.ambr +++ /dev/null @@ -1,387 +0,0 @@ -# serializer version: 1 -# name: test_flowmap_export_matched - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - ]) -# --- -# name: test_flowmap_export_unmatched - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Cesium-134', - 'unit': 'kBq', - }), - dict({ - 'categories': list([ - 'Emissions to soil', - '', - ]), - 'name': 'Cesium-134', - 'unit': 'kBq', - }), - dict({ - 'categories': list([ - 'Resources', - 'in ground', - ]), - 'name': 'Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, as N', - 'unit': 'kg', - }), - ]) -# --- -# name: test_flowmap_mappings_ei39_ei310 - list([ - dict({ - 'comment': 'Identical uuid', - 'conversion_factor': 1.0, - 'source': dict({ - '@id': '4f777e05-70f9-4a18-a406-d8232325073f', - 'compartment': dict({ - '@subcompartmentId': 'be7e06e9-0bf5-462e-99dc-fe4aee383c48', - 'compartment': dict({ - '#text': 'air', - '@xml:lang': 'en', - }), - 'subcompartment': dict({ - '#text': 'non-urban air or from high stacks', - '@xml:lang': 'en', - }), - }), - 'name': dict({ - '#text': '2,4-D amines', - '@xml:lang': 'en', - }), - 'unitName': dict({ - '#text': 'kg', - '@xml:lang': 'en', - }), - }), - 'target': dict({ - 'context': 'air/non-urban air or from high stacks', - 'name': '2,4-D dimethylamine salt', - 'unit': 'kg', - 'uuid': 'b6b4201e-0561-5992-912f-e729fbf04e41', - }), - }), - ]) -# --- -# name: test_flowmap_mappings_ei_ei - list([ - dict({ - 'comment': 'Identical uuid', - 'conversion_factor': 1.0, - 'source': dict({ - '@id': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - 'compartment': dict({ - '@subcompartmentId': '7011f0aa-f5f9-4901-8c10-884ad8296812', - 'compartment': dict({ - '#text': 'air', - '@xml:lang': 'en', - }), - 'subcompartment': dict({ - '#text': 'unspecified', - '@xml:lang': 'en', - }), - }), - 'name': dict({ - '#text': '1,4-Butanediol', - '@xml:lang': 'en', - }), - 'unitName': dict({ - '#text': 'kg', - '@xml:lang': 'en', - }), - }), - 'target': dict({ - 'context': 'air/unspecified', - 'name': '1,4-Butanediol', - 'unit': 'kg', - 'uuid': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - }), - }), - dict({ - 'comment': 'Identical uuid', - 'conversion_factor': 1.0, - 'source': dict({ - '@id': '0f440cc0-0f74-446d-99d6-8ff0e97a2444', - 'compartment': dict({ - '@subcompartmentId': 'be7e06e9-0bf5-462e-99dc-fe4aee383c48', - 'compartment': dict({ - '#text': 'air', - '@xml:lang': 'en', - }), - 'subcompartment': dict({ - '#text': 'non-urban air or from high stacks', - '@xml:lang': 'en', - }), - }), - 'name': dict({ - '#text': 'Ammonia', - '@xml:lang': 'en', - }), - 'unitName': dict({ - '#text': 'kg', - '@xml:lang': 'en', - }), - }), - 'target': dict({ - 'context': 'air/non-urban air or from high stacks', - 'name': 'Ammonia', - 'unit': 'kg', - 'uuid': '0f440cc0-0f74-446d-99d6-8ff0e97a2444', - }), - }), - ]) -# --- -# name: test_flowmap_nomatch_multiple_rules - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - ]) -# --- -# name: test_flowmap_nomatch_rule - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - ]) -# --- -# name: test_flowmap_nomatch_rule_false - list([ - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'CAS': '000110-63-4', - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Cesium-134', - 'unit': 'kBq', - }), - dict({ - 'categories': list([ - 'Emissions to soil', - '', - ]), - 'name': 'Cesium-134', - 'unit': 'kBq', - }), - dict({ - 'categories': list([ - 'Resources', - 'in ground', - ]), - 'name': 'Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, as N', - 'unit': 'kg', - }), - ]) -# --- -# name: test_flowmap_remove_duplicates - list([ - 1,4-Butanediol (in kg) [uuid:None], - 1,4-Butanediol (in kg) [uuid:None], - Cesium-134 (in kBq) [uuid:None], - Cesium-134 (in kBq) [uuid:None], - Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore (in kg) [uuid:None], - Ammonia, FR (in kg) [uuid:None], - Ammonia, as N (in kg) [uuid:None], - ]) -# --- -# name: test_flowmap_to_randonneur - list([ - dict({ - 'comment': 'Identical names', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/unspecified', - 'name': '1,4-Butanediol', - 'unit': 'kg', - 'uuid': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - }), - }), - dict({ - 'comment': 'Names with country code', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/non-urban air or from high stacks', - 'location': 'FR', - 'name': 'Ammonia', - 'unit': 'kg', - 'uuid': '0f440cc0-0f74-446d-99d6-8ff0e97a2444', - }), - }), - ]) -# --- -# name: test_flowmap_to_randonneur_export - list([ - dict({ - 'comment': 'Identical names', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/unspecified', - 'name': '1,4-Butanediol', - 'unit': 'kg', - 'uuid': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - }), - }), - dict({ - 'comment': 'Names with country code', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/non-urban air or from high stacks', - 'location': 'FR', - 'name': 'Ammonia', - 'unit': 'kg', - 'uuid': '0f440cc0-0f74-446d-99d6-8ff0e97a2444', - }), - }), - ]) -# --- -# name: test_flowmap_with_custom_rules_match - list([ - dict({ - 'comment': 'Identical names', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/unspecified', - 'name': '1,4-Butanediol', - 'unit': 'kg', - 'uuid': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - }), - }), - ]) -# --- -# name: test_flowmap_with_custom_rules_no_match - list([ - ]) -# --- diff --git a/tests/__snapshots__/test_read_migration_files.ambr b/tests/__snapshots__/test_read_migration_files.ambr deleted file mode 100644 index 57f95b6..0000000 --- a/tests/__snapshots__/test_read_migration_files.ambr +++ /dev/null @@ -1,101 +0,0 @@ -# serializer version: 1 -# name: test_read_multiple_migration_files - dict({ - 'update': list([ - dict({ - 'source': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Cesium-134', - 'unit': 'kBq', - }), - 'target': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop., long-term', - ]), - }), - }), - dict({ - 'source': dict({ - 'name': 'Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore', - }), - 'target': dict({ - 'name': 'Zinc, Zn 0.63%, in mixed ore', - }), - }), - dict({ - 'source': dict({ - '@id': '4f777e05-70f9-4a18-a406-d8232325073f', - }), - 'target': dict({ - '@id': 'b6b4201e-0561-5992-912f-e729fbf04e41', - }), - }), - dict({ - 'source': dict({ - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - }), - 'target': dict({ - 'categories': list([ - 'Air', - '(unspecified)', - ]), - }), - }), - dict({ - 'conversion_factor': 1.2142857142857142, - 'source': dict({ - 'name': 'Ammonia, as N', - }), - 'target': dict({ - 'name': 'Ammonia', - }), - }), - ]), - }) -# --- -# name: test_read_single_migration_file - dict({ - 'update': list([ - dict({ - 'source': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Cesium-134', - 'unit': 'kBq', - }), - 'target': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop., long-term', - ]), - }), - }), - dict({ - 'source': dict({ - 'name': 'Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore', - }), - 'target': dict({ - 'name': 'Zinc, Zn 0.63%, in mixed ore', - }), - }), - dict({ - 'source': dict({ - '@id': '4f777e05-70f9-4a18-a406-d8232325073f', - }), - 'target': dict({ - '@id': 'b6b4201e-0561-5992-912f-e729fbf04e41', - }), - }), - ]), - }) -# --- diff --git a/tests/__snapshots__/test_transform_flow.ambr b/tests/__snapshots__/test_transform_flow.ambr deleted file mode 100644 index 88d53a8..0000000 --- a/tests/__snapshots__/test_transform_flow.ambr +++ /dev/null @@ -1,78 +0,0 @@ -# serializer version: 1 -# name: test_transform_flow_with_conversion_factor - list([ - dict({ - 'comment': 'Identical names', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Air', - '(unspecified)', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/unspecified', - 'name': '1,4-Butanediol', - 'unit': 'kg', - 'uuid': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - }), - }), - dict({ - 'comment': 'Identical names', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Air', - 'high. pop.', - ]), - 'name': '1,4-Butanediol', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/unspecified', - 'name': '1,4-Butanediol', - 'unit': 'kg', - 'uuid': '09db39be-d9a6-4fc3-8d25-1f80b23e9131', - }), - }), - dict({ - 'comment': 'Identical names', - 'conversion_factor': 1.2142857142857142, - 'source': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, as N', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/non-urban air or from high stacks', - 'name': 'Ammonia', - 'unit': 'kg', - 'uuid': '0f440cc0-0f74-446d-99d6-8ff0e97a2444', - }), - }), - dict({ - 'comment': 'Names with country code', - 'conversion_factor': 1.0, - 'source': dict({ - 'categories': list([ - 'Emissions to air', - 'low. pop.', - ]), - 'name': 'Ammonia, FR', - 'unit': 'kg', - }), - 'target': dict({ - 'context': 'air/non-urban air or from high stacks', - 'location': 'FR', - 'name': 'Ammonia', - 'unit': 'kg', - 'uuid': '0f440cc0-0f74-446d-99d6-8ff0e97a2444', - }), - }), - ]) -# --- diff --git a/tests/conftest.py b/tests/conftest.py index 007aeee..3d1a949 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,92 +1,16 @@ """Fixtures for flowmapper""" import pytest -from flowmapper.utils import read_field_mapping, read_flowlist, read_migration_files -from flowmapper.flow import Flow -@pytest.fixture -def field_mapping(): - result = { - "source": { - "name": "name", - "context": "categories", - "unit": "unit", - }, - "target": { - "uuid": "@id", - "context": "compartment.*.#text", - "name": "name.#text", - "synonyms": ("synonym", ["#text"]), - "unit": "unitName.#text", - }, - } - return result - - -@pytest.fixture -def fields(): - result = { - "source": { - "uuid": "Flow UUID", - "name": "Flowable", - "context": "Context", - "unit": "Unit", - "cas": "CAS No", - }, - "target": { - "uuid": "FlowUUID", - "name": "Flowable", - "context": "Context", - "unit": "Unit", - "cas": "CASNo", - }, - } - - return result - - -@pytest.fixture -def source_flows(): - fields = read_field_mapping("tests/data/field_mapping-sp-ei.py") - result = [ - Flow(flow, fields["source"]) for flow in read_flowlist("tests/data/sp.json") - ] - return result - - -@pytest.fixture -def target_flows(): - fields = read_field_mapping("tests/data/field_mapping-sp-ei.py") - result = [ - Flow(flow, fields["target"]) for flow in read_flowlist("tests/data/ei-3.7.json") - ] - return result - - -@pytest.fixture -def flows_ei39(): - transformations = read_migration_files("tests/data/migrations.json") - fields = { - "uuid": "@id", - "name": "name.#text", - "synonyms": ("synonym", ["#text"]), - "context": "compartment.*.#text", - "unit": "unitName.#text", - "cas": "@casNumber", - } - result = [Flow(flow, fields, transformations) for flow in read_flowlist("tests/data/ei-3.9.json")] - return result +from flowmapper.domain import Flow +from flowmapper.transformation_mapping import prepare_transformations +from flowmapper.utils import ( + apply_transformations, + load_standard_transformations, + read_migration_files, +) @pytest.fixture -def flows_ei310(): - fields = { - "uuid": "@id", - "name": "name.#text", - "synonyms": ("synonym", ["#text"]), - "context": "compartment.*.#text", - "unit": "unitName.#text", - "cas": "@casNumber", - } - result = [Flow(flow, fields) for flow in read_flowlist("tests/data/ei-3.10.json")] - return result +def transformations(): + return prepare_transformations(load_standard_transformations()) diff --git a/tests/data/ei-3.10.json b/tests/data/ei-3.10.json index 93a7723..b62837e 100644 --- a/tests/data/ei-3.10.json +++ b/tests/data/ei-3.10.json @@ -1,27 +1,9 @@ [ { - "@id": "b6b4201e-0561-5992-912f-e729fbf04e41", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@formula": "C10H13Cl2NO3", - "@casNumber": "002008-39-1", - "name": { - "@xml:lang": "en", - "#text": "2,4-D dimethylamine salt" - }, - "unitName": { - "@xml:lang": "en", - "#text": "kg" - }, - "compartment": { - "@subcompartmentId": "be7e06e9-0bf5-462e-99dc-fe4aee383c48", - "compartment": { - "@xml:lang": "en", - "#text": "air" - }, - "subcompartment": { - "@xml:lang": "en", - "#text": "non-urban air or from high stacks" - } - } -} -] \ No newline at end of file + "identifier": "b6b4201e-0561-5992-912f-e729fbf04e41", + "cas_number": "002008-39-1", + "name": "2,4-D dimethylamine salt", + "unit": "kg", + "context": ["air", "non-urban air or from high stacks"] + } +] diff --git a/tests/data/ei-3.7.json b/tests/data/ei-3.7.json index 5bd4aba..dec80c2 100644 --- a/tests/data/ei-3.7.json +++ b/tests/data/ei-3.7.json @@ -1,54 +1,17 @@ [ - { - "@id": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@casNumber": "000110-63-4", - "name": { - "@xml:lang": "en", - "#text": "1,4-Butanediol" - }, - "unitName": { - "@xml:lang": "en", - "#text": "kg" - }, - "compartment": { - "@subcompartmentId": "7011f0aa-f5f9-4901-8c10-884ad8296812", - "compartment": { - "@xml:lang": "en", - "#text": "air" - }, - "subcompartment": { - "@xml:lang": "en", - "#text": "unspecified" - } - }, - "synonym": { - "@xml:lang": "en", - "#text": "Butylene glycol" - } - }, - { - "@id": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@casNumber": "007664-41-7", - "name": { - "@xml:lang": "en", - "#text": "Ammonia" - }, - "unitName": { - "@xml:lang": "en", - "#text": "kg" - }, - "compartment": { - "@subcompartmentId": "be7e06e9-0bf5-462e-99dc-fe4aee383c48", - "compartment": { - "@xml:lang": "en", - "#text": "air" - }, - "subcompartment": { - "@xml:lang": "en", - "#text": "non-urban air or from high stacks" - } - } - } -] \ No newline at end of file + { + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "cas_number": "000110-63-4", + "name": "1,4-Butanediol", + "unit": "kg", + "context": ["air", "unspecified"], + "synonym": "Butylene glycol" + }, + { + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "cas_number": "007664-41-7", + "name": "Ammonia", + "unit": "kg", + "context": ["air", "non-urban air or from high stacks"] + } +] diff --git a/tests/data/ei-3.9.json b/tests/data/ei-3.9.json index 2105808..6d2c1e1 100644 --- a/tests/data/ei-3.9.json +++ b/tests/data/ei-3.9.json @@ -1,41 +1,15 @@ [ { - "@id": "4f777e05-70f9-4a18-a406-d8232325073f", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@formula": "C10H13Cl2NO3", - "@casNumber": "002008-39-1", - "name": { - "@xml:lang": "en", - "#text": "2,4-D amines" - }, - "unitName": { - "@xml:lang": "en", - "#text": "kg" - }, - "compartment": { - "@subcompartmentId": "be7e06e9-0bf5-462e-99dc-fe4aee383c48", - "compartment": { - "@xml:lang": "en", - "#text": "air" - }, - "subcompartment": { - "@xml:lang": "en", - "#text": "non-urban air or from high stacks" - } - }, - "synonym": [ - { - "@xml:lang": "en", - "#text": "2-(2,4-dichlorophenoxy)acetic acid" - }, - { - "@xml:lang": "en", - "#text": "2,4-D dimethylamine salt" - }, - { - "@xml:lang": "en", - "#text": "N-methylmethanamine" - } + "identifier": "4f777e05-70f9-4a18-a406-d8232325073f", + "formula": "C10H13Cl2NO3", + "cas_number": "002008-39-1", + "name": "2,4-D amines", + "unit": "kg", + "context": ["air", "non-urban air or from high stacks"], + "synonyms": [ + "2-(2,4-dichlorophenoxy)acetic acid", + "2,4-D dimethylamine salt", + "N-methylmethanamine" ] } -] \ No newline at end of file +] diff --git a/tests/data/field_mapping-sp-ei.py b/tests/data/field_mapping-sp-ei.py deleted file mode 100644 index a80e01b..0000000 --- a/tests/data/field_mapping-sp-ei.py +++ /dev/null @@ -1,8 +0,0 @@ -config = { - "uuid": ["", "@id"], - "name": ["name", "name.#text"], - "synonyms": ["", ("synonym", ["#text"])], - "context": ["categories", "compartment.*.#text"], - "unit": ["unit", "unitName.#text"], - "cas": ["CAS", "@casNumber"], -} diff --git a/tests/data/migrations.json b/tests/data/migrations.json index f37d394..71a972f 100644 --- a/tests/data/migrations.json +++ b/tests/data/migrations.json @@ -1,24 +1,36 @@ -[ - { - "target": {"categories": ["Emissions to air", "low. pop., long-term"]}, +{ + "update": [ + { + "target": { + "context": [ + "Emissions to air", + "low. pop., long-term" + ] + }, "source": { - "name": "Cesium-134", - "categories": ["Emissions to air", "low. pop."], - "unit": "kBq" + "name": "Cesium-134", + "context": [ + "Emissions to air", + "low. pop." + ], + "unit": "kBq" } - }, - { - "target": {"name": "Zinc, Zn 0.63%, in mixed ore"}, + }, + { + "target": { + "name": "Zinc, Zn 0.63%, in mixed ore" + }, "source": { - "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore" + "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore" } - }, - { - "source": { - "@id": "4f777e05-70f9-4a18-a406-d8232325073f" }, - "target": { - "@id": "b6b4201e-0561-5992-912f-e729fbf04e41" + { + "source": { + "identifier": "4f777e05-70f9-4a18-a406-d8232325073f" + }, + "target": { + "identifier": "b6b4201e-0561-5992-912f-e729fbf04e41" + } } - } -] + ] +} \ No newline at end of file diff --git a/tests/data/sp.json b/tests/data/sp.json index d9adc12..4a9bef8 100644 --- a/tests/data/sp.json +++ b/tests/data/sp.json @@ -1,69 +1,48 @@ [ - { - "name": "1,4-Butanediol", - "categories": [ - "Air", - "(unspecified)" - ], - "unit": "kg", - "CAS": "000110-63-4" - }, - { + { "name": "1,4-Butanediol", - "categories": [ - "Air", - "(unspecified)" - ], + "context": "air", "unit": "kg", - "CAS": "000110-63-4" - }, - { - "name": "1,4-Butanediol", - "categories": [ - "Air", - "high. pop." - ], - "unit": "kg", - "CAS": "000110-63-4" - }, - { - "name": "Cesium-134", - "unit": "kBq", - "categories": [ - "Emissions to air", - "low. pop." - ] + "cas_number": "000110-63-4" }, - { - "name": "Cesium-134", - "unit": "kBq", - "categories": [ - "Emissions to soil", - "" - ] - }, - { - "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore", + { + "name": "1,4-Butanediol", + "context": "air", "unit": "kg", - "categories": [ - "Resources", - "in ground" - ] + "cas_number": "000110-63-4" + }, + { + "name": "1,4-Butanediol", + "context": "air/high. pop.", + "unit": "kg", + "cas_number": "000110-63-4" + }, + { + "name": "Cesium-134", + "unit": "kBq", + "context": "air/low. pop." + }, + { + "name": "Cesium-134", + "unit": "kBq", + "context": [ + "Emissions to soil", + "" + ] + }, + { + "name": "Zinc, Zn 0.63%, Au 9.7E-4%, Ag 9.7E-4%, Cu 0.38%, Pb 0.014%, in ore", + "unit": "kg", + "context": "resources/in ground" }, { "name": "Ammonia, FR", "unit": "kg", - "categories": [ - "Emissions to air", - "low. pop." - ] + "context": "air/low. pop." }, { "name": "Ammonia, as N", "unit": "kg", - "categories": [ - "Emissions to air", - "low. pop." - ] + "context": "air/low. pop." } ] diff --git a/tests/data/transformations.json b/tests/data/transformations.json index 3e86397..a566642 100644 --- a/tests/data/transformations.json +++ b/tests/data/transformations.json @@ -1,18 +1,22 @@ -[ - { - "target": { "categories": ["Air", "(unspecified)"] }, - "source": { - "name": "1,4-Butanediol", - "categories": ["Air", "high. pop."] - } - }, - { - "source": { - "name": "Ammonia, as N" - }, - "target": { - "name": "Ammonia" +{ + "update": [ + { + "source": { + "name": "1,4-Butanediol", + "context": "air/high. pop." + }, + "target": { + "context": ["Air", "(unspecified)"] + } }, - "conversion_factor": 1.2142857142857142 - } -] + { + "source": { + "name": "Ammonia, as N" + }, + "target": { + "name": "Ammonia" + }, + "conversion_factor": 1.2142857142857142 + } + ] +} \ No newline at end of file diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..d33f44e --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests for flowmapper using real objects.""" diff --git a/tests/integration/test_match_integration.py b/tests/integration/test_match_integration.py new file mode 100644 index 0000000..272b045 --- /dev/null +++ b/tests/integration/test_match_integration.py @@ -0,0 +1,514 @@ +"""Integration tests for match.py functions using real Flow objects.""" + +import pytest + +from flowmapper.domain import Flow +from flowmapper.match import ( + match_biogenic_to_non_fossil, + match_custom_names_with_location_codes, + match_emissions_with_suffix_ion, + match_flows_with_suffix_unspecified_origin, + match_names_with_location_codes, + match_names_with_roman_numerals_in_parentheses, + match_non_ionic_state, + match_resource_names_with_location_codes_and_parent_context, + match_resources_with_suffix_in_air, + match_resources_with_suffix_in_ground, + match_resources_with_suffix_in_water, + match_resources_with_wrong_subcontext, + match_rules, +) + + +class TestMatchNamesWithRomanNumeralsInParentheses: + """Integration tests for match_names_with_roman_numerals_in_parentheses.""" + + def test_match_names_with_roman_numerals_in_parentheses_matching( + self, transformations + ): + """Test matching names with roman numerals in parentheses.""" + source = { + "name": "Iron (ii)", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Iron ii", + "context": ["air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_names_with_roman_numerals_in_parentheses(s, t, [], []) + + assert result == {"comment": "With/without roman numerals in parentheses"} + + def test_match_names_with_roman_numerals_in_parentheses_uppercase( + self, transformations + ): + """Test matching names with uppercase roman numerals in parentheses.""" + source = { + "name": "Iron (II)", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Iron II", + "context": ["air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_names_with_roman_numerals_in_parentheses(s, t, [], []) + + assert result == {"comment": "With/without roman numerals in parentheses"} + + def test_match_names_with_roman_numerals_in_parentheses_mixed_case( + self, transformations + ): + """Test matching names with mixed case roman numerals in parentheses.""" + source = { + "name": "Iron (II)", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Iron ii", + "context": ["air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_names_with_roman_numerals_in_parentheses(s, t, [], []) + + assert result == {"comment": "With/without roman numerals in parentheses"} + + def test_match_names_with_roman_numerals_in_parentheses_no_match( + self, transformations + ): + """Test when names don't match even after removing roman numerals.""" + source = { + "name": "Iron (II)", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Copper", + "context": ["air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_names_with_roman_numerals_in_parentheses(s, t, [], []) + + assert result is None + + def test_match_names_with_roman_numerals_in_parentheses_different_context( + self, transformations + ): + """Test when contexts are different.""" + source = { + "name": "Iron (II)", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Iron", + "context": ["ground"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_names_with_roman_numerals_in_parentheses(s, t, [], []) + + assert result is None + + +class TestMatchResourceNamesWithLocationCodesAndParentContext: + """Integration tests for match_resource_names_with_location_codes_and_parent_context.""" + + def test_match_resource_names_with_location_codes_and_parent_context_matching( + self, transformations + ): + """Test matching resource names with location codes and parent context.""" + source = { + "name": "Water, NL", + "context": ["natural resource", "in air"], + "unit": "kg", + } + target = { + "name": "Water", + "context": ["natural resource", "in air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resource_names_with_location_codes_and_parent_context( + s, t, [], [] + ) + + assert result is not None + assert ( + result["comment"] == "Name matching with location code and parent context" + ) + assert result["location"] == "NL" + + def test_match_resource_names_with_location_codes_water_conversion( + self, transformations + ): + """Test water conversion factor for resource names with location codes.""" + source = { + "name": "Water, NL", + "context": ["natural resource", "in air"], + "unit": "cubic_meter", + } + target = { + "name": "Water", + "context": ["natural resource", "in air"], + "unit": "kilogram", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resource_names_with_location_codes_and_parent_context( + s, t, [], [] + ) + + assert result is not None + assert result["conversion_factor"] == 1000.0 + + def test_match_resource_names_with_location_codes_no_match(self, transformations): + """Test when resource names don't match.""" + source = { + "name": "Water, NL", + "context": ["natural resource", "in air"], + "unit": "kg", + } + target = { + "name": "Air", + "context": ["natural resource", "in air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resource_names_with_location_codes_and_parent_context( + s, t, [], [] + ) + + assert result is None + + +class TestMatchResourcesWithSuffixInGround: + """Integration tests for match_resources_with_suffix_in_ground.""" + + def test_match_resources_with_suffix_in_ground_matching(self, transformations): + """Test matching resources with suffix 'in ground'.""" + source = { + "name": "Copper", + "context": ["natural resource", "in ground"], + "unit": "kg", + } + target = { + "name": "Copper, in ground", + "context": ["natural resource", "in ground"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resources_with_suffix_in_ground(s, t, [], []) + + assert result == {"comment": "Resources with suffix in ground"} + + def test_match_resources_with_suffix_in_ground_no_match(self, transformations): + """Test when resources don't match.""" + source = { + "name": "Copper", + "context": ["natural resource", "in ground"], + "unit": "kg", + } + target = { + "name": "Iron, in ground", + "context": ["natural resource", "in ground"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resources_with_suffix_in_ground(s, t, [], []) + + assert result is None + + +class TestMatchFlowsWithSuffixUnspecifiedOrigin: + """Integration tests for match_flows_with_suffix_unspecified_origin.""" + + def test_match_flows_with_suffix_unspecified_origin_matching(self, transformations): + """Test matching flows with suffix 'unspecified origin'.""" + source = { + "name": "Carbon dioxide", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Carbon dioxide, unspecified origin", + "context": ["air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_flows_with_suffix_unspecified_origin(s, t, [], []) + + assert result == {"comment": "Flows with suffix unspecified origin"} + + def test_match_flows_with_suffix_unspecified_origin_no_match(self, transformations): + """Test when flows don't match.""" + source = { + "name": "Carbon dioxide", + "context": ["air"], + "unit": "kg", + } + target = { + "name": "Methane, unspecified origin", + "context": ["air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_flows_with_suffix_unspecified_origin(s, t, [], []) + + assert result is None + + +class TestMatchResourcesWithSuffixInWater: + """Integration tests for match_resources_with_suffix_in_water.""" + + def test_match_resources_with_suffix_in_water_matching(self, transformations): + """Test matching resources with suffix 'in water'.""" + source = { + "name": "Copper", + "context": ["natural resource", "in water"], + "unit": "kg", + } + target = { + "name": "Copper, in water", + "context": ["natural resource", "in water"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resources_with_suffix_in_water(s, t, [], []) + + assert result == {"comment": "Resources with suffix in water"} + + def test_match_resources_with_suffix_in_water_no_match(self, transformations): + """Test when resources don't match.""" + source = { + "name": "Copper", + "context": ["natural resource", "in water"], + "unit": "kg", + } + target = { + "name": "Iron, in water", + "context": ["natural resource", "in water"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resources_with_suffix_in_water(s, t, [], []) + + assert result is None + + +class TestMatchResourcesWithSuffixInAir: + """Integration tests for match_resources_with_suffix_in_air.""" + + def test_match_resources_with_suffix_in_air_matching(self, transformations): + """Test matching resources with suffix 'in air'.""" + source = { + "name": "Nitrogen", + "context": ["natural resource", "in air"], + "unit": "kg", + } + target = { + "name": "Nitrogen, in air", + "context": ["natural resource", "in air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resources_with_suffix_in_air(s, t, [], []) + + assert result == {"comment": "Resources with suffix in air"} + + def test_match_resources_with_suffix_in_air_no_match(self, transformations): + """Test when resources don't match.""" + source = { + "name": "Nitrogen", + "context": ["natural resource", "in air"], + "unit": "kg", + } + target = { + "name": "Oxygen, in air", + "context": ["natural resource", "in air"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_resources_with_suffix_in_air(s, t, [], []) + + assert result is None + + +class TestMatchEmissionsWithSuffixIon: + """Integration tests for match_emissions_with_suffix_ion.""" + + def test_match_emissions_with_suffix_ion_matching(self, transformations): + """Test matching emissions with suffix 'ion'.""" + source = { + "name": "Copper", + "context": ["emission", "to water"], + "unit": "kg", + } + target = { + "name": "Copper, ion", + "context": ["emission", "to water"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_emissions_with_suffix_ion(s, t, [], []) + + assert result == {"comment": "Match emissions with suffix ion"} + + def test_match_emissions_with_suffix_ion_no_match(self, transformations): + """Test when emissions don't match.""" + source = { + "name": "Copper", + "context": ["emission", "to water"], + "unit": "kg", + } + target = { + "name": "Iron, ion", + "context": ["emission", "to water"], + "unit": "kg", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + result = match_emissions_with_suffix_ion(s, t, [], []) + + assert result is None, f"Expected result to be None, but got {result}" + + +class TestMatchRules: + """Integration tests for match_rules function.""" + + def test_match_rules_returns_list(self): + """Test that match_rules returns a list of functions.""" + rules = match_rules() + + assert isinstance(rules, list) + assert len(rules) > 0 + assert all(callable(rule) for rule in rules) + + def test_match_rules_contains_expected_functions(self): + """Test that match_rules contains expected matching functions.""" + from flowmapper.match import ( + match_biogenic_to_non_fossil, + match_custom_names_with_location_codes, + match_emissions_with_suffix_ion, + match_flows_with_suffix_unspecified_origin, + match_identical_cas_numbers, + match_identical_identifier, + match_identical_names, + match_identical_names_in_preferred_synonyms, + match_identical_names_in_synonyms, + match_identical_names_without_commas, + match_names_with_location_codes, + match_names_with_roman_numerals_in_parentheses, + match_non_ionic_state, + match_resource_names_with_location_codes_and_parent_context, + match_resources_with_suffix_in_air, + match_resources_with_suffix_in_ground, + match_resources_with_suffix_in_water, + match_resources_with_wrong_subcontext, + ) + + rules = match_rules() + + assert match_identical_identifier in rules + assert match_identical_names in rules + assert match_identical_names_without_commas in rules + assert match_resources_with_suffix_in_ground in rules + assert match_resources_with_suffix_in_water in rules + assert match_resources_with_suffix_in_air in rules + assert match_flows_with_suffix_unspecified_origin in rules + assert match_resources_with_wrong_subcontext in rules + assert match_emissions_with_suffix_ion in rules + assert match_names_with_roman_numerals_in_parentheses in rules + assert match_names_with_location_codes in rules + assert match_resource_names_with_location_codes_and_parent_context in rules + assert match_custom_names_with_location_codes in rules + assert match_identical_cas_numbers in rules + assert match_non_ionic_state in rules + assert match_biogenic_to_non_fossil in rules + assert match_identical_names_in_preferred_synonyms in rules + assert match_identical_names_in_synonyms in rules + + def test_match_rules_order(self): + """Test that match_rules returns functions in expected order.""" + rules = match_rules() + + # Check that some key functions are in the expected order + rule_names = [rule.__name__ for rule in rules] + + # match_identical_identifier should be first + assert ( + rule_names[0] == "match_identical_identifier" + ), f"Expected rule_names[0] to be 'match_identical_identifier', but got {rule_names[0]!r}" + + # match_identical_names should be early + assert ( + "match_identical_names" in rule_names[:5] + ), f"Expected 'match_identical_names' to be in rule_names[:5], but got {rule_names[:5]}" + + # More complex matches should be later + assert ( + "match_custom_names_with_location_codes" in rule_names + ), f"Expected 'match_custom_names_with_location_codes' to be in rule_names, but it was not" + assert ( + "match_biogenic_to_non_fossil" in rule_names[-5:] + ), f"Expected 'match_biogenic_to_non_fossil' to be in rule_names[-5:], but got {rule_names[-5:]}" diff --git a/tests/test_cas.py b/tests/test_cas.py deleted file mode 100644 index fc823e3..0000000 --- a/tests/test_cas.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -from flowmapper.cas import CAS - - -def test_cas_format(): - assert CAS('96-49-1') == CAS('0000096-49-1') - -def test_invalid_cas(): - assert not CAS('96-49-2').valid - assert CAS('96-49-2').check_digit_expected == 1 - -def test_integer_cas(): - with pytest.raises(TypeError): - CAS(96491) - -def test_cas_repr(): - repr('0000096-49-1') == '96-49-1' - -def test_equality_comparison_with_missing_cas(): - assert not CAS('') == CAS('') - -def test_equality_comparison_with_newlines(): - assert CAS('\t\n\n007440-05-3') == CAS('7440-05-3') diff --git a/tests/test_cli.py b/tests/test_cli.py index fbeeb55..d0e5caa 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,17 @@ +import json + from typer.testing import CliRunner + from flowmapper.cli import app -import json -import pandas as pd runner = CliRunner() def test_version(): result = runner.invoke(app, ["--version"]) - assert result.output.startswith("flowmapper, version") + assert result.output.startswith( + "flowmapper, version" + ), f"Expected result.output to start with 'flowmapper, version', but got {result.output[:50]!r}" def test_format_glad(tmp_path): @@ -20,8 +23,6 @@ def test_format_glad(tmp_path): "tests/data/ei-3.7.json", "--format", "glad", - "--fields", - "tests/data/field_mapping-sp-ei.py", "--output-dir", str(tmp_path), ], @@ -36,8 +37,12 @@ def test_format_glad(tmp_path): files = sorted(tmp_path.glob("**/*")) - assert result.exit_code == 0 - assert expected_files == files + assert ( + result.exit_code == 0 + ), f"Expected exit_code to be 0, but got {result.exit_code}" + assert ( + expected_files == files + ), f"Expected files to be {expected_files}, but got {files}" def test_format_randonneur(tmp_path): @@ -49,8 +54,6 @@ def test_format_randonneur(tmp_path): "tests/data/ei-3.7.json", "--format", "randonneur", - "--fields", - "tests/data/field_mapping-sp-ei.py", "--output-dir", str(tmp_path), ], @@ -65,11 +68,15 @@ def test_format_randonneur(tmp_path): files = sorted(tmp_path.glob("**/*")) - assert result.exit_code == 0 - assert expected_files == files + assert ( + result.exit_code == 0 + ), f"Expected exit_code to be 0, but got {result.exit_code}" + assert ( + expected_files == files + ), f"Expected files to be {expected_files}, but got {files}" -def test_matched_flows(tmp_path, snapshot): +def test_matched_flows(tmp_path): runner.invoke( app, [ @@ -78,8 +85,6 @@ def test_matched_flows(tmp_path, snapshot): "tests/data/ei-3.7.json", "--matched-source", "--matched-target", - "--fields", - "tests/data/field_mapping-sp-ei.py", "--output-dir", str(tmp_path), ], @@ -88,10 +93,21 @@ def test_matched_flows(tmp_path, snapshot): with open(tmp_path / "sp-ei-3.7-matched-source.json") as fs: actual = json.load(fs) - assert actual == snapshot - - -def test_matched_flows_with_randonneur_transformations(tmp_path, snapshot): + expected = [ + { + "cas_number": "110-63-4", + "context": "air", + "name": "1,4-Butanediol", + "unit": "kg", + }, + {"context": "air/low. pop.", "name": "Ammonia, FR", "unit": "kg"}, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" + + +def test_matched_flows_with_randonneur_transformations(tmp_path): runner.invoke( app, [ @@ -102,8 +118,6 @@ def test_matched_flows_with_randonneur_transformations(tmp_path, snapshot): "tests/data/transformations.json", "--matched-source", "--matched-target", - "--fields", - "tests/data/field_mapping-sp-ei.py", "--output-dir", str(tmp_path), ], @@ -112,10 +126,28 @@ def test_matched_flows_with_randonneur_transformations(tmp_path, snapshot): with open(tmp_path / "sp-ei-3.7-matched-source.json") as fs: actual = json.load(fs) - assert actual == snapshot - - -def test_matched_flows_with_multiple_randonneur_transformations(tmp_path, snapshot): + expected = [ + { + "cas_number": "110-63-4", + "context": "air", + "name": "1,4-Butanediol", + "unit": "kg", + }, + { + "cas_number": "110-63-4", + "context": "air/high. pop.", + "name": "1,4-Butanediol", + "unit": "kg", + }, + {"context": "air/low. pop.", "name": "Ammonia, FR", "unit": "kg"}, + {"context": "air/low. pop.", "name": "Ammonia, as N", "unit": "kg"}, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" + + +def test_matched_flows_with_multiple_randonneur_transformations(tmp_path): runner.invoke( app, [ @@ -128,8 +160,6 @@ def test_matched_flows_with_multiple_randonneur_transformations(tmp_path, snapsh "tests/data/migrations.json", "--matched-source", "--matched-target", - "--fields", - "tests/data/field_mapping-sp-ei.py", "--output-dir", str(tmp_path), ], @@ -138,4 +168,22 @@ def test_matched_flows_with_multiple_randonneur_transformations(tmp_path, snapsh with open(tmp_path / "sp-ei-3.7-matched-source.json") as fs: actual = json.load(fs) - assert actual == snapshot + expected = [ + { + "name": "1,4-Butanediol", + "unit": "kg", + "context": "air", + "cas_number": "110-63-4", + }, + { + "name": "1,4-Butanediol", + "unit": "kg", + "context": "air/high. pop.", + "cas_number": "110-63-4", + }, + {"name": "Ammonia, FR", "unit": "kg", "context": "air/low. pop."}, + {"name": "Ammonia, as N", "unit": "kg", "context": "air/low. pop."}, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" diff --git a/tests/test_context.py b/tests/test_context.py deleted file mode 100644 index 25c62a7..0000000 --- a/tests/test_context.py +++ /dev/null @@ -1,73 +0,0 @@ -from flowmapper.context import Context - - -def test_context(): - data = { - "name": "Carbon dioxide, in air", - "categories": ["Raw", "(unspecified)"], - "unit": "kg", - "CAS": "000124-38-9", - } - fields = "categories" - actual = Context.from_dict(data, fields).to_dict() - expected = { - "value": "natural resource/in ground", - "raw_value": "Raw/(unspecified)", - "raw_object": {"categories": ["Raw", "(unspecified)"]}, - } - assert actual == expected - - -def test_trailing_slash(): - c1 = Context.from_dict({"categories": ["Raw", "(unspecified)"]}, "categories") - c2 = Context.from_dict({"categories": ["Raw"]}, "categories") - c3 = Context.from_dict({"categories": ["Raw/"]}, "categories") - assert c1.value == c2.value - assert c2.value == c3.value - "/".join(c1.value) - - -def test_unspecified(): - c1 = Context.from_dict( - { - "compartment": { - "@subcompartmentId": "e47f0a6c-3be8-4027-9eee-de251784f708", - "compartment": {"@xml:lang": "en", "#text": "water"}, - "subcompartment": {"@xml:lang": "en", "#text": "unspecified"}, - }, - }, - "compartment.*.#text", - ) - - c2 = Context.from_dict({"categories": ["Emissions to water", ""]}, "categories") - c3 = Context.from_dict({"categories": ["Water", "(unspecified)"]}, "categories") - c4 = Context.from_dict({"categories": ["Water", ""]}, "categories") - c5 = Context.from_dict({"categories": ["Water/", ""]}, "categories") - c6 = Context.from_dict({"categories": ["Water/"]}, "categories") - c7 = Context.from_dict({"categories": "Water/(unspecified)"}, "categories") - c8 = Context.from_dict({"categories": "Water/unspecified"}, "categories") - c9 = Context.from_dict({"categories": "Water/"}, "categories") - c10 = Context.from_dict({"categories": "Water"}, "categories") - c11 = Context.from_dict( - {"context": [{"name": "Water"}, {"name": "unspecified"}]}, ("context", ["name"]) - ) - c12 = Context.from_dict({"categories": ["Water"]}, "categories") - - actual = set( - [ - c1.value, - c2.value, - c3.value, - c4.value, - c5.value, - c6.value, - c7.value, - c8.value, - c9.value, - c10.value, - c11.value, - c12.value, - ] - ) - expected = {"water"} - assert actual == expected diff --git a/tests/test_extract_country_code.py b/tests/test_extract_country_code.py deleted file mode 100644 index bf5c02d..0000000 --- a/tests/test_extract_country_code.py +++ /dev/null @@ -1,13 +0,0 @@ -from flowmapper.utils import extract_country_code - -def test_with_valid_country_code(): - assert extract_country_code('ammonia, nl') == ('ammonia', 'nl') - -def test_with_no_country_code(): - assert extract_country_code('ammonia') == ('ammonia', None) - -def test_with_additional_text(): - assert extract_country_code('ammonia, nl something') == ('ammonia, nl something', None) - -def test_with_space_before_country_code(): - assert extract_country_code('ammonia, nl') == ('ammonia', 'nl') diff --git a/tests/test_extract_ecospold2.py b/tests/test_extract_ecospold2.py new file mode 100644 index 0000000..e88bc4a --- /dev/null +++ b/tests/test_extract_ecospold2.py @@ -0,0 +1,241 @@ +from flowmapper.extraction.ecospold2 import remove_conflicting_synonyms + + +def test_remove_conflicting_synonyms_no_conflicts(): + """Test that synonyms are kept when there are no conflicts.""" + data = [ + {"name": "flow_a", "context": ["ground"], "synonyms": ["water", "h2o"]}, + {"name": "flow_b", "context": ["ground"], "synonyms": ["soil", "earth"]}, + ] + + result = remove_conflicting_synonyms(data) + + assert result[0]["synonyms"] == [ + "water", + "h2o", + ], f"Expected result[0]['synonyms'] to equal ['water', 'h2o'], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "soil", + "earth", + ], f"Expected result[1]['synonyms'] to equal ['soil', 'earth'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_with_conflicts(): + """Test that conflicting synonyms are removed.""" + data = [ + {"name": "flow_a", "context": ["ground"], "synonyms": ["water", "h2o"]}, + { + "name": "water", # This conflicts with the synonym in flow_a + "context": ["ground"], + "synonyms": ["aqua"], + }, + ] + + result = remove_conflicting_synonyms(data) + + # "water" should be removed from flow_a's synonyms + assert result[0]["synonyms"] == [ + "h2o" + ], f"Expected result[0]['synonyms'] to equal ['h2o'], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_different_contexts(): + """Test that synonyms are kept when conflicts exist in different contexts.""" + data = [ + {"name": "flow_a", "context": ["ground"], "synonyms": ["water", "h2o"]}, + { + "name": "water", # This conflicts but is in different context + "context": ["air"], + "synonyms": ["aqua"], + }, + ] + + result = remove_conflicting_synonyms(data) + + # "water" should be kept since contexts are different + assert result[0]["synonyms"] == [ + "water", + "h2o", + ], f"Expected result[0]['synonyms'] to equal ['water', 'h2o'], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_multiple_conflicts(): + """Test removal of multiple conflicting synonyms.""" + data = [ + {"name": "flow_a", "context": ["ground"], "synonyms": ["water", "soil", "h2o"]}, + {"name": "water", "context": ["ground"], "synonyms": ["aqua"]}, + {"name": "soil", "context": ["ground", "deep"], "synonyms": ["earth"]}, + ] + + result = remove_conflicting_synonyms(data) + + # Both "water" and "soil" should be removed from flow_a's synonyms + assert result[0]["synonyms"] == [ + "h2o" + ], f"Expected result[0]['synonyms'] to equal ['h2o'], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" + assert result[2]["synonyms"] == [ + "earth" + ], f"Expected result[2]['synonyms'] to equal ['earth'], but got {result[2]['synonyms']}" + + +def test_remove_conflicting_synonyms_no_synonyms(): + """Test handling of flows without synonyms.""" + data = [ + { + "name": "flow_a", + "context": ["ground"], + # No synonyms + }, + {"name": "flow_b", "context": ["ground"], "synonyms": ["water"]}, + ] + + result = remove_conflicting_synonyms(data) + + # Should not raise error and flow_b should keep its synonym + assert ( + "synonyms" not in result[0] + ), "Expected 'synonyms' to not be in result[0], but it was" + assert result[1]["synonyms"] == [ + "water" + ], f"Expected result[1]['synonyms'] to equal ['water'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_no_context(): + """Test handling of flows without context.""" + data = [ + { + "name": "flow_a", + "synonyms": ["water", "h2o"], + # No context + }, + {"name": "water", "context": ["ground"], "synonyms": ["aqua"]}, + ] + + result = remove_conflicting_synonyms(data) + + # flow_a should keep its synonyms since it has no context + assert result[0]["synonyms"] == [ + "water", + "h2o", + ], f"Expected result[0]['synonyms'] to equal ['water', 'h2o'], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_empty_synonyms_list(): + """Test handling of empty synonyms list.""" + data = [ + {"name": "flow_a", "context": ["ground"], "synonyms": []}, + {"name": "water", "context": ["ground"], "synonyms": ["aqua"]}, + ] + + result = remove_conflicting_synonyms(data) + + # Empty synonyms list should remain empty + assert ( + result[0]["synonyms"] == [] + ), f"Expected result[0]['synonyms'] to equal [], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_case_insensitive(): + """Test that synonym removal is case insensitive.""" + data = [ + {"name": "flow_a", "context": ["ground"], "synonyms": ["Water", "H2O"]}, + { + "name": "water", # Lowercase, different from "Water" + "context": ["ground"], + "synonyms": ["aqua"], + }, + ] + + result = remove_conflicting_synonyms(data) + + assert result[0]["synonyms"] == [ + "H2O" + ], f"Expected result[0]['synonyms'] to equal ['H2O'], but got {result[0]['synonyms']}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" + + +def test_remove_conflicting_synonyms_self_conflict(): + """Test that a flow's own name doesn't conflict with its synonyms.""" + data = [ + {"name": "water", "context": ["ground"], "synonyms": ["h2o", "aqua", "water"]} + ] + + result = remove_conflicting_synonyms(data) + + # All synonyms should be kept since they don't conflict with other flows + assert result[0]["synonyms"] == [ + "h2o", + "aqua", + ], f"Expected result[0]['synonyms'] to equal ['h2o', 'aqua'], but got {result[0]['synonyms']}" + + +def test_remove_conflicting_synonyms_preserves_original_data(): + """Test that the function doesn't modify other fields in the data.""" + data = [ + { + "name": "flow_a", + "context": ["ground"], + "synonyms": ["water", "h2o"], + "unit": "kg", + "identifier": "123", + }, + { + "name": "water", + "context": ["ground"], + "synonyms": ["aqua"], + "unit": "m3", + "identifier": "456", + }, + ] + + result = remove_conflicting_synonyms(data) + + # Check that other fields are preserved + assert ( + result[0]["name"] == "flow_a" + ), f"Expected result[0]['name'] to equal 'flow_a', but got {result[0]['name']!r}" + assert result[0]["context"] == [ + "ground" + ], f"Expected result[0]['context'] to equal ['ground'], but got {result[0]['context']}" + assert ( + result[0]["unit"] == "kg" + ), f"Expected result[0]['unit'] to equal 'kg', but got {result[0]['unit']!r}" + assert ( + result[0]["identifier"] == "123" + ), f"Expected result[0]['identifier'] to equal '123', but got {result[0]['identifier']!r}" + assert result[0]["synonyms"] == [ + "h2o" + ], f"Expected result[0]['synonyms'] to equal ['h2o'], but got {result[0]['synonyms']}" # Only "water" removed + + assert ( + result[1]["name"] == "water" + ), f"Expected result[1]['name'] to equal 'water', but got {result[1]['name']!r}" + assert result[1]["context"] == [ + "ground" + ], f"Expected result[1]['context'] to equal ['ground'], but got {result[1]['context']}" + assert ( + result[1]["unit"] == "m3" + ), f"Expected result[1]['unit'] to equal 'm3', but got {result[1]['unit']!r}" + assert ( + result[1]["identifier"] == "456" + ), f"Expected result[1]['identifier'] to equal '456', but got {result[1]['identifier']!r}" + assert result[1]["synonyms"] == [ + "aqua" + ], f"Expected result[1]['synonyms'] to equal ['aqua'], but got {result[1]['synonyms']}" diff --git a/tests/test_flow.py b/tests/test_flow.py index e2e1e43..ede39a3 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -1,5 +1,7 @@ -from flowmapper.cas import CAS -from flowmapper.flow import Flow +from flowmapper.cas import CASField +from flowmapper.domain import Flow +from flowmapper.transformation_mapping import prepare_transformations + def test_flow_with_transformations_repr(): d = { @@ -9,171 +11,157 @@ def test_flow_with_transformations_repr(): "cas": "000124-38-9", } - transformations = {"update": [ - { - "source": {"name": "Carbon dioxide, in air", "context": ["Raw", "(unspecified)"]}, - "target": {"name": "Carbon dioxide"}, - } - ]} + transformations = prepare_transformations( + [ + { + "update": [ + { + "source": { + "name": "Carbon dioxide, in air", + "context": ["Raw", "(unspecified)"], + }, + "target": {"name": "Carbon dioxide"}, + } + ] + } + ] + ) f = Flow(d, transformations=transformations) - - assert repr(f) == "Carbon dioxide, in air* (in kg) [uuid:None]" + expected = """Flow object: + Identifier: StringField with missing original value + Name: StringField: 'Carbon dioxide, in air' -> 'carbon dioxide' + Context: ContextField: '['Raw', '(unspecified)']' -> '('raw',)' + Unit: UnitField: 'kg' -> 'kg'""" + assert ( + repr(f) == expected + ), f"Expected repr(f) to equal expected string, but got {repr(f)!r} instead of {expected!r}" -def test_flow_with_jsonpath_expr(): - data = { - "@id": "87883a4e-1e3e-4c9d-90c0-f1bea36f8014", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@casNumber": "007664-41-7", - "name": {"@xml:lang": "en", "#text": "Ammonia"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "7011f0aa-f5f9-4901-8c10-884ad8296812", - "compartment": {"@xml:lang": "en", "#text": "air"}, - "subcompartment": {"@xml:lang": "en", "#text": "unspecified"}, - }, - } - fields = { - "cas": "@casNumber", - "context": "compartment.*.#text", - "name": "name.#text", - "unit": "unitName.#text", - "uuid": "@id", - } - - flow = Flow(data, fields) - assert flow.name.value == "ammonia" - assert flow.name.raw_value == "Ammonia" - assert flow.name.raw_object == {"name": {"@xml:lang": "en", "#text": "Ammonia"}} - assert not flow.synonyms - assert flow.context.value == "air" - assert flow.unit.value == "kilogram" - assert flow.unit.raw_value == "kg" - assert flow.unit.raw_object == {"unitName": {"@xml:lang": "en", "#text": "kg"}} - - -def test_flow_from_sp_categories(): +def test_flow_from_sp_categories(transformations): data = { "name": "Carbon dioxide, in air", - "categories": ["Raw", "(unspecified)"], + "context": "resources/in air", "unit": "kg", - "CAS": "000124-38-9", - } - - fields = { - "name": "name", - "context": "categories", - "unit": "unit", - "cas": "CAS", - } - - flow = Flow(data, fields) - assert flow.uuid is None - assert flow.name.raw_value == "Carbon dioxide, in air" - assert flow.context.raw_value == "Raw/(unspecified)" - assert id(flow.raw) == id(data) - - -def test_flow_from_sp_missing(): - data = {"name": "Chrysotile", "context": ["Resources", "in ground"], "unit": "kg"} - - fields = { - "name": "name", - "context": "context", - "unit": "unit", + "cas_number": "000124-38-9", } - flow = Flow(data, fields) - assert flow.name.raw_value == "Chrysotile" - assert repr(flow.cas) == "" - assert flow.context.raw_value == "Resources/in ground" - - -def test_flow_from_sp(): + flow = Flow(data, transformations) + assert ( + not flow.identifier + ), f"Expected flow.identifier to be falsy, but got {flow.identifier}" + assert ( + flow.name.original == "Carbon dioxide, in air" + ), f"Expected flow.name.original to be 'Carbon dioxide, in air', but got {flow.name.original!r}" + assert ( + flow.name.normalized == "carbon dioxide, in air" + ), f"Expected flow.name.normalized to be 'carbon dioxide, in air', but got {flow.name.normalized!r}" + assert ( + flow.context.original == "resources/in air" + ), f"Expected flow.context.original to be 'resources/in air', but got {flow.context.original!r}" + assert flow.context.normalized == ( + "natural resource", + "in air", + ), f"Expected flow.context.normalized to be ('natural resource', 'in air'), but got {flow.context.normalized!r}" + + +def test_flow_from_sp_missing(transformations): + data = {"name": "Chrysotile", "context": "Raw/in ground", "unit": "kg"} + + flow = Flow(data, transformations) + assert ( + flow.name.original == "Chrysotile" + ), f"Expected flow.name.original to be 'Chrysotile', but got {flow.name.original!r}" + expected = """Flow object: + Identifier: StringField with missing original value + Name: StringField: 'Chrysotile' -> 'chrysotile' + Context: ContextField: 'Raw/in ground' -> '('natural resource', 'in ground')' + Unit: UnitField: 'kg' -> 'kg'""" + assert ( + repr(flow) == expected + ), f"Expected repr(flow) to equal expected string, but got {repr(flow)!r} instead of {expected!r}" + assert ( + flow.context.original == "Raw/in ground" + ), f"Expected flow.context.original to be 'Raw/in ground', but got {flow.context.original!r}" + assert flow.context.normalized == ( + "natural resource", + "in ground", + ), f"Expected flow.context.normalized to be ('natural resource', 'in ground'), but got {flow.context.normalized!r}" + + +def test_flow_cas(): data = { - "Flowable": "Actinium", - "CAS No": "007440-34-8", - "Formula": "Ac\u007f", - "Synonyms": "Actinium", - "Unit": "kg", + "name": "Actinium", + "cas_number": "007440-34-8", + "chemical formula": "Ac\u007f", + "synonyms": "Actinium", + "unit": "kg", "Class": "Raw materials", - "Context": "Raw materials", - "Flow UUID": "90004354-71D3-47E8-B322-300BA5A98F7B", + "context": "Raw materials", "Description": "", } fields = { - "uuid": "Flow UUID", - "name": "Flowable", - "context": "Context", - "unit": "Unit", - "cas": "CAS No", + "identifier": "Flow UUID", + "name": "name", + "context": "context", + "unit": "unit", + "cas_number": "CAS No", } - flow = Flow(data, fields) - assert flow.uuid == "90004354-71D3-47E8-B322-300BA5A98F7B" - assert flow.cas == CAS("007440-34-8") - assert flow.cas.cas == "7440-34-8" + flow = Flow(data) + assert flow.cas_number == CASField( + "007440-34-8" + ), f"Expected flow.cas to equal CASField('007440-34-8'), but got {flow.cas_number!r}" + assert ( + flow.cas_number == "7440-34-8" + ), f"Expected flow.cas to equal '7440-34-8', but got {flow.cas_number!r}" def test_flow_from_ei(): data = { - "Flowable": "1,3-Dioxolan-2-one", - "CASNo": "000096-49-1", - "Formula": "", - "Synonyms": "", - "Unit": "kg", + "name": "1,3-Dioxolan-2-one", + "cas_number": "000096-49-1", + "chemical formula": "", + "synonyms": "", + "unit": "kg", "Class": "chemical", "ExternalReference": "", "Preferred": "", - "Context": "water/unspecified", - "FlowUUID": "5b7d620e-2238-5ec9-888a-6999218b6974", + "context": "water/unspecified", + "identifier": "5b7d620e-2238-5ec9-888a-6999218b6974", "AltUnit": "", "Var": "", "Second CAS": "96-49-1", } + flow = Flow(data) + assert ( + flow.identifier == "5b7d620e-2238-5ec9-888a-6999218b6974" + ), f"Expected flow.identifier to be '5b7d620e-2238-5ec9-888a-6999218b6974', but got {flow.identifier!r}" - fields = { - "uuid": "FlowUUID", - "name": "Flowable", - "context": "Context", - "unit": "Unit", - "cas": "CASNo", - } - - flow = Flow(data, fields) - assert flow.uuid == "5b7d620e-2238-5ec9-888a-6999218b6974" - -def test_flow_with_synonyms(field_mapping): +def test_flow_with_synonyms(transformations): data = { - "@id": "f0cc0453-32c0-48f5-b8d4-fc87d100b8d9", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@casNumber": "000078-79-5", - "name": {"@xml:lang": "en", "#text": "Isoprene"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "23dbff79-8037-43e7-b270-5a3da416a284", - "compartment": {"@xml:lang": "en", "#text": "air"}, - "subcompartment": { - "@xml:lang": "en", - "#text": "low population density, long-term", - }, - }, - "synonym": [ - {"@xml:lang": "en", "#text": "2-methylbuta-1,3-diene"}, - {"@xml:lang": "en", "#text": "methyl bivinyl"}, - {"@xml:lang": "en", "#text": "hemiterpene"}, + "identifier": "f0cc0453-32c0-48f5-b8d4-fc87d100b8d9", + "cas_number": "000078-79-5", + "name": "Isoprene", + "unit": "kg", + "context": ["air", "low population density, long-term"], + "synonyms": [ + "2-methylbuta-1,3-diene", + "methyl bivinyl", + "hemiterpene", ], } - flow = Flow(data, field_mapping["target"]) - assert flow.synonyms.raw_value == [ + flow = Flow(data, transformations) + actual_synonyms = [obj.original for obj in flow.synonyms] + expected_synonyms = [ "2-methylbuta-1,3-diene", "methyl bivinyl", "hemiterpene", ] - - + assert ( + actual_synonyms == expected_synonyms + ), f"Expected flow.synonyms to be {expected_synonyms}, but got {actual_synonyms}" diff --git a/tests/test_flowmap.py b/tests/test_flowmap.py index 7c0b31e..bd243dd 100644 --- a/tests/test_flowmap.py +++ b/tests/test_flowmap.py @@ -1,20 +1,54 @@ -import pandas as pd import json +from pathlib import Path + +import pandas as pd +import pytest + from flowmapper import Flowmap -from flowmapper.match import ( - match_emissions_with_suffix_ion, - match_identical_names, -) +from flowmapper.domain import Flow +from flowmapper.match import match_emissions_with_suffix_ion, match_identical_names + +DATA_DIR = Path(__file__).parent / "data" + + +@pytest.fixture +def source_flows(transformations): + return [ + Flow(flow, transformations) for flow in json.load(open(DATA_DIR / "sp.json")) + ] + -def test_flowmap_remove_duplicates(source_flows, target_flows, snapshot): +@pytest.fixture +def target_flows(transformations): + return [ + Flow(flow, transformations) + for flow in json.load(open(DATA_DIR / "ei-3.7.json")) + ] + + +@pytest.fixture +def ei39(): + return [Flow(flow) for flow in json.load(open(DATA_DIR / "ei-3.9.json"))] + + +@pytest.fixture +def ei310(): + return [Flow(flow) for flow in json.load(open(DATA_DIR / "ei-3.10.json"))] + + +def test_flowmap_remove_duplicates(source_flows, target_flows): flowmap = Flowmap(source_flows, target_flows) actual = flowmap.source_flows - assert actual == snapshot + # Added one duplicate on purpose + assert ( + len(flowmap.source_flows) == 7 + ), f"Expected len(flowmap.source_flows) to be 7, but got {len(flowmap.source_flows)}" + def test_flowmap_mappings(source_flows, target_flows): flowmap = Flowmap(source_flows, target_flows) actual = flowmap.mappings[0] - assert list(actual.keys()) == [ + expected_keys = [ "from", "to", "conversion_factor", @@ -22,35 +56,210 @@ def test_flowmap_mappings(source_flows, target_flows): "match_rule_priority", "info", ] - assert actual["match_rule"] == "match_identical_names" + assert ( + list(actual.keys()) == expected_keys + ), f"Expected actual.keys() to be {expected_keys}, but got {list(actual.keys())}" + assert ( + actual["match_rule"] == "match_identical_names" + ), f"Expected actual['match_rule'] to be 'match_identical_names', but got {actual['match_rule']!r}" -def test_flowmap_to_randonneur(source_flows, target_flows, snapshot): +def test_flowmap_to_randonneur(source_flows, target_flows): flowmap = Flowmap(source_flows, target_flows) - actual = flowmap.to_randonneur() - assert actual == snapshot + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + ) + actual = dp.data["update"] + expected = [ + { + "comment": "Identical names", + "conversion_factor": 1.0, + "source": { + "cas_number": "110-63-4", + "context": "air", + "name": "1,4-Butanediol", + "unit": "kg", + }, + "target": { + "cas_number": "110-63-4", + "context": ["air", "unspecified"], + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "name": "1,4-Butanediol", + "unit": "kg", + }, + }, + { + "comment": "Name matching with location code", + "conversion_factor": 1.0, + "location": "FR", + "source": {"context": "air/low. pop.", "name": "Ammonia, FR", "unit": "kg"}, + "target": { + "cas_number": "7664-41-7", + "context": ["air", "non-urban air or from high stacks"], + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "name": "Ammonia", + "unit": "kg", + }, + }, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" + -def test_flowmap_to_randonneur_export(source_flows, target_flows, snapshot, tmp_path): +def test_flowmap_to_randonneur_export(source_flows, target_flows, tmp_path): flowmap = Flowmap(source_flows, target_flows) - flowmap.to_randonneur(tmp_path / "randonneur.json") - with open(tmp_path / "randonneur.json", "r") as fs: - actual = json.load(fs) - assert actual == snapshot + flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + path=tmp_path / "randonneur.json", + ) + with open(tmp_path / "randonneur.json") as fs: + data = json.load(fs) + actual = data["update"] + expected = [ + { + "comment": "Identical names", + "conversion_factor": 1.0, + "source": { + "cas_number": "110-63-4", + "context": "air", + "name": "1,4-Butanediol", + "unit": "kg", + }, + "target": { + "cas_number": "110-63-4", + "context": ["air", "unspecified"], + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "name": "1,4-Butanediol", + "unit": "kg", + }, + }, + { + "comment": "Name matching with location code", + "conversion_factor": 1.0, + "location": "FR", + "source": {"context": "air/low. pop.", "name": "Ammonia, FR", "unit": "kg"}, + "target": { + "cas_number": "7664-41-7", + "context": ["air", "non-urban air or from high stacks"], + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "name": "Ammonia", + "unit": "kg", + }, + }, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" + -def test_flowmap_with_custom_rules_no_match(source_flows, target_flows, snapshot): +def test_flowmap_with_custom_rules_no_match(source_flows, target_flows): flowmap = Flowmap( source_flows, target_flows, rules=[match_emissions_with_suffix_ion], ) actual = flowmap.mappings - assert actual == snapshot + assert actual == [], f"Expected actual to be an empty list, but got {actual}" -def test_flowmap_with_custom_rules_match(source_flows, target_flows, snapshot): +def test_flowmap_with_custom_rules_match(source_flows, target_flows): flowmap = Flowmap(source_flows, target_flows, rules=[match_identical_names]) - actual = flowmap.to_randonneur() - assert actual == snapshot + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + ) + actual = dp.data["update"] + expected = [ + { + "comment": "Identical names", + "conversion_factor": 1.0, + "source": { + "cas_number": "110-63-4", + "context": "air", + "name": "1,4-Butanediol", + "unit": "kg", + }, + "target": { + "cas_number": "110-63-4", + "context": [ + "air", + "unspecified", + ], + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "name": "1,4-Butanediol", + "unit": "kg", + }, + } + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" def test_flowmap_to_glad(source_flows, target_flows): @@ -58,10 +267,10 @@ def test_flowmap_to_glad(source_flows, target_flows): actual = flowmap.to_glad() expected = { "SourceFlowName": ["1,4-Butanediol", "Ammonia, FR"], - "SourceFlowUUID": [None, None], - "SourceFlowContext": ["Air/(unspecified)", "Emissions to air/low. pop."], + "SourceFlowUUID": ["", ""], + "SourceFlowContext": ["air", "air/low. pop."], "SourceUnit": ["kg", "kg"], - "MatchCondition": ["", ""], + "MatchCondition": ["=", "="], "ConversionFactor": [1.0, 1.0], "TargetFlowName": ["1,4-Butanediol", "Ammonia"], "TargetFlowUUID": [ @@ -69,24 +278,26 @@ def test_flowmap_to_glad(source_flows, target_flows): "0f440cc0-0f74-446d-99d6-8ff0e97a2444", ], "TargetFlowContext": [ - "air/unspecified", - "air/non-urban air or from high stacks", + "air✂️unspecified", + "air✂️non-urban air or from high stacks", ], "TargetUnit": ["kg", "kg"], - "MemoMapper": ["Identical names", "Names with country code"], + "MemoMapper": ["Identical names", "Name matching with location code"], } - assert actual.equals(pd.DataFrame(expected)) + pd.testing.assert_frame_equal(actual, pd.DataFrame(expected)) + def test_flowmap_to_glad_export(source_flows, target_flows, tmp_path): flowmap = Flowmap(source_flows, target_flows) flowmap.to_glad(tmp_path / "glad.xlsx") actual = pd.read_excel(tmp_path / "glad.xlsx") + print(actual["MatchCondition"]) expected = { "SourceFlowName": ["1,4-Butanediol", "Ammonia, FR"], - "SourceFlowUUID": [float("nan"), float("nan")], - "SourceFlowContext": ["Air/(unspecified)", "Emissions to air/low. pop."], + "SourceFlowUUID": [float("NaN"), float("NaN")], + "SourceFlowContext": ["air", "air/low. pop."], "SourceUnit": ["kg", "kg"], - "MatchCondition": [float("nan"), float("nan")], + "MatchCondition": ["=", "="], "ConversionFactor": [1, 1], "TargetFlowName": ["1,4-Butanediol", "Ammonia"], "TargetFlowUUID": [ @@ -94,52 +305,184 @@ def test_flowmap_to_glad_export(source_flows, target_flows, tmp_path): "0f440cc0-0f74-446d-99d6-8ff0e97a2444", ], "TargetFlowContext": [ - "air/unspecified", - "air/non-urban air or from high stacks", + "air✂️unspecified", + "air✂️non-urban air or from high stacks", ], "TargetUnit": ["kg", "kg"], - "MemoMapper": ["Identical names", "Names with country code"], + "MemoMapper": ["Identical names", "Name matching with location code"], } - assert actual.equals(pd.DataFrame(expected)) - -def test_flowmap_export_matched(source_flows, target_flows, snapshot): - flowmap = Flowmap(source_flows, target_flows) - actual = [flow.raw for flow in flowmap.matched_source] - assert actual == snapshot + pd.testing.assert_frame_equal(actual, pd.DataFrame(expected)) -def test_flowmap_export_unmatched(source_flows, target_flows, snapshot): - flowmap = Flowmap(source_flows, target_flows) - actual = [flow.raw for flow in flowmap.unmatched_source] - assert actual == snapshot -def test_flowmap_nomatch_rule(source_flows, target_flows, snapshot): - nomatch = lambda flow: flow.context.value == "air/urban air close to ground" +def test_flowmap_nomatch_rule(source_flows, target_flows): + nomatch = lambda flow: flow.context == "air/urban air close to ground" flowmap = Flowmap(source_flows, target_flows, nomatch_rules=[nomatch]) - actual = [flow.raw for flow in flowmap.source_flows_nomatch] - assert actual == snapshot + assert ( + len(flowmap.source_flows_nomatch) == 1 + ), f"Expected len(flowmap.source_flows_nomatch) to be 1, but got {len(flowmap.source_flows_nomatch)}" + assert ( + flowmap.source_flows_nomatch[0].name == "1,4-Butanediol" + ), f"Expected flowmap.source_flows_nomatch[0].name to be '1,4-Butanediol', but got {flowmap.source_flows_nomatch[0].name!r}" + assert ( + flowmap.source_flows_nomatch[0].context == "air/urban air close to ground" + ), f"Expected flowmap.source_flows_nomatch[0].context to be 'air/urban air close to ground', but got {flowmap.source_flows_nomatch[0].context!r}" + assert ( + flowmap.source_flows[0].name == "1,4-Butanediol" + ), f"Expected flowmap.source_flows[0].name to be '1,4-Butanediol', but got {flowmap.source_flows[0].name!r}" + assert ( + flowmap.source_flows[0].context == "air" + ), f"Expected flowmap.source_flows[0].context to be 'air', but got {flowmap.source_flows[0].context!r}" + -def test_flowmap_nomatch_rule_false(source_flows, target_flows, snapshot): - nomatch = lambda flow: flow.context.value == "water" +def test_flowmap_nomatch_rule_false(source_flows, target_flows): + nomatch = lambda flow: flow.context == "water" flowmap = Flowmap(source_flows, target_flows, nomatch_rules=[nomatch]) + assert ( + not flowmap.source_flows_nomatch + ), f"Expected flowmap.source_flows_nomatch to be falsy, but got {flowmap.source_flows_nomatch}" - actual = [flow.raw for flow in flowmap.source_flows] - assert actual == snapshot -def test_flowmap_nomatch_multiple_rules(source_flows, target_flows, snapshot): - nomatch1 = lambda flow: flow.context.value == "air/urban air close to ground" - nomatch2 = lambda flow: flow.context.value == "air" +def test_flowmap_nomatch_multiple_rules(source_flows, target_flows): + nomatch1 = lambda flow: flow.context == "air/urban air close to ground" + nomatch2 = lambda flow: flow.context == "air" flowmap = Flowmap(source_flows, target_flows, nomatch_rules=[nomatch1, nomatch2]) - actual = [flow.raw for flow in flowmap.source_flows_nomatch] - assert actual == snapshot + assert ( + len(flowmap.source_flows_nomatch) == 2 + ), f"Expected len(flowmap.source_flows_nomatch) to be 2, but got {len(flowmap.source_flows_nomatch)}" + assert ( + flowmap.source_flows_nomatch[0].name == "1,4-Butanediol" + ), f"Expected flowmap.source_flows_nomatch[0].name to be '1,4-Butanediol', but got {flowmap.source_flows_nomatch[0].name!r}" + assert ( + flowmap.source_flows_nomatch[1].name == "1,4-Butanediol" + ), f"Expected flowmap.source_flows_nomatch[1].name to be '1,4-Butanediol', but got {flowmap.source_flows_nomatch[1].name!r}" + assert ( + flowmap.source_flows[0].name == "Cesium-134" + ), f"Expected flowmap.source_flows[0].name to be 'Cesium-134', but got {flowmap.source_flows[0].name!r}" -def test_flowmap_mappings_ei_ei(target_flows, snapshot): + +def test_flowmap_mappings_ei_ei(target_flows): flowmap = Flowmap(target_flows, target_flows) - actual = flowmap.to_randonneur() - assert actual == snapshot + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + ) + actual = dp.data["update"] + expected = [ + { + "source": { + "name": "1,4-Butanediol", + "unit": "kg", + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "context": ["air", "unspecified"], + "cas_number": "110-63-4", + }, + "target": { + "name": "1,4-Butanediol", + "unit": "kg", + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "context": ["air", "unspecified"], + "cas_number": "110-63-4", + }, + "conversion_factor": 1.0, + "comment": "Identical identifier", + }, + { + "source": { + "name": "Ammonia", + "unit": "kg", + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "context": ["air", "non-urban air or from high stacks"], + "cas_number": "7664-41-7", + }, + "target": { + "name": "Ammonia", + "unit": "kg", + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "context": ["air", "non-urban air or from high stacks"], + "cas_number": "7664-41-7", + }, + "conversion_factor": 1.0, + "comment": "Identical identifier", + }, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" + -def test_flowmap_mappings_ei39_ei310(flows_ei39, flows_ei310, snapshot): - flowmap = Flowmap(flows_ei39, flows_ei310) - actual = flowmap.to_randonneur() - assert actual == snapshot +def test_flowmap_mappings_ei39_ei310(ei39, ei310): + flowmap = Flowmap(ei39, ei310) + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + ) + actual = dp.data["update"] + expected = [ + { + "source": { + "name": "2,4-D amines", + "unit": "kg", + "identifier": "4f777e05-70f9-4a18-a406-d8232325073f", + "context": ["air", "non-urban air or from high stacks"], + "cas_number": "2008-39-1", + }, + "target": { + "name": "2,4-D dimethylamine salt", + "unit": "kg", + "identifier": "b6b4201e-0561-5992-912f-e729fbf04e41", + "context": ["air", "non-urban air or from high stacks"], + "cas_number": "2008-39-1", + }, + "conversion_factor": 1.0, + "comment": "Identical CAS numbers", + } + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" diff --git a/tests/test_flowproperty.py b/tests/test_flowproperty.py deleted file mode 100644 index 6de5672..0000000 --- a/tests/test_flowproperty.py +++ /dev/null @@ -1,50 +0,0 @@ -from flowmapper.flowproperty import FlowProperty - -def test_empty(): - p1 = FlowProperty("") - p2 = FlowProperty("") - assert not p1 - assert p1 != p2 - -def test_list_with_falsy_values(): - p1 = FlowProperty(["", "FOO", None]) - p1 == ["foo"] - -def test_get(): - d = {'foo': 1, "FOO": 2} - p = FlowProperty("FOO") - assert d.get(p) == 1 - -def test_equals(): - p1 = FlowProperty("foo") - p2 = FlowProperty("FOO") - p3 = FlowProperty() - assert p1 == "foo" - assert p2 == "foo" - assert p2 != "FOO" - assert p1 == p2 - assert p1 != p3 - - p3 = FlowProperty(["foo", "bar"]) - p4 = FlowProperty(["FOO", "BAR"]) - assert p3 == ["foo", "bar"] - assert p4 == ["foo", "bar"] - assert p4 != ["FOO", "BAR"] - assert p3 == p4 - -def test_init(): - n = FlowProperty.from_dict({"flowable": "\u0055\u0308ber"}, "flowable") - assert n.value == "\u00FCber" - assert n.raw_value == "\u0055\u0308ber" - assert n.raw_object == {"flowable": "\u0055\u0308ber"} - assert n.to_dict() == {'value': 'über', 'raw_value': 'Über', 'raw_object': {'flowable': 'Über'}} - - -def test_init_synonyms(): - s = FlowProperty.from_dict({ - "synonym": [ - {"@xml:lang": "en", "#text": "Cumol"}, - {"@xml:lang": "en", "#text": "isopropylbenzene"}, - ] - }, ("synonym", ["#text"])) - diff --git a/tests/test_format_match_result.py b/tests/test_format_match_result.py deleted file mode 100644 index fe60c3d..0000000 --- a/tests/test_format_match_result.py +++ /dev/null @@ -1,47 +0,0 @@ -from deepdiff import DeepDiff -from flowmapper.match import format_match_result -from flowmapper.flow import Flow - -def test_format_match_result_missing_id(): - source = { - "name": "Carbon dioxide, in air", - "context": "Raw materials", - "unit": "kg", - } - - source_fields = { - "uuid": "", - "name": "name", - "context": "context", - "unit": "unit", - } - - s = Flow(source, source_fields) - - target = { - "id": "cc6a1abb-b123-4ca6-8f16-38209df609be", - "name": "Carbon dioxide, in air", - "context": "natural resource/in air", - "unit": "kg", - } - - target_fields = { - "uuid": "id", - "name": "name", - "context": "context", - "unit": "unit", - } - - t = Flow(target, target_fields) - - actual = format_match_result(s, t, 1.0, {'is_match': True, 'comment': 'foo'}) - expected = {'source': {'name': 'Carbon dioxide, in air', 'context': 'Raw materials', 'unit': 'kg'}, - 'target': {'uuid': 'cc6a1abb-b123-4ca6-8f16-38209df609be', - 'name': 'Carbon dioxide, in air', - 'context': 'natural resource/in air', - 'unit': 'kg'}, - 'conversion_factor': 1.0, - 'comment': 'foo'} - - diff = DeepDiff(actual, expected) - assert not diff diff --git a/tests/test_get_conversion_factor.py b/tests/test_get_conversion_factor.py index a71abe5..de481e0 100644 --- a/tests/test_get_conversion_factor.py +++ b/tests/test_get_conversion_factor.py @@ -1,151 +1,124 @@ -from flowmapper.flow import Flow import math +from flowmapper.domain import Flow -def test_get_conversion_factor(field_mapping): + +def test_get_conversion_factor(transformations): s = Flow( { "name": "Protactinium-234", "unit": "Bq", - "categories": ["Emissions to air", "low. pop."], + "context": ["Emissions to air", "low. pop."], }, - field_mapping["source"], + transformations, ) t = Flow( { - "@id": "fb13070e-06f1-4964-832f-a23945b880cc", - "@unitId": "4923348e-591b-4772-b224-d19df86f04c9", - "name": {"@xml:lang": "en", "#text": "Protactinium-234"}, - "unitName": {"@xml:lang": "en", "#text": "kBq"}, - "compartment": { - "@subcompartmentId": "be7e06e9-0bf5-462e-99dc-fe4aee383c48", - "compartment": {"@xml:lang": "en", "#text": "air"}, - "subcompartment": { - "@xml:lang": "en", - "#text": "non-urban air or from high stacks", - }, - }, + "identifier": "fb13070e-06f1-4964-832f-a23945b880cc", + "name": "Protactinium-234", + "unit": "kBq", + "context": ["air", "non-urban air or from high stacks"], }, - field_mapping["target"], + transformations, ) actual = s.unit.conversion_factor(t.unit) expected = 1e-3 - assert actual == expected + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" -def test_get_conversion_factor_water(field_mapping): +def test_get_conversion_factor_water(transformations): s = Flow( - {"name": "Water", "unit": "kg", "categories": ["Emissions to water", ""]}, - field_mapping["source"], + {"name": "Water", "unit": "kg", "context": ["Emissions to water", ""]}, + transformations, ) t = Flow( { - "@id": "2404b41a-2eed-4e9d-8ab6-783946fdf5d6", - "@unitId": "de5b3c87-0e35-4fb0-9765-4f3ba34c99e5", - "@casNumber": "007732-18-5", - "name": {"@xml:lang": "en", "#text": "Water"}, - "unitName": {"@xml:lang": "en", "#text": "m3"}, - "compartment": { - "@subcompartmentId": "e47f0a6c-3be8-4027-9eee-de251784f708", - "compartment": {"@xml:lang": "en", "#text": "water"}, - "subcompartment": {"@xml:lang": "en", "#text": "unspecified"}, - }, + "identifier": "2404b41a-2eed-4e9d-8ab6-783946fdf5d6", + "cas_number": "007732-18-5", + "name": "Water", + "unit": "m3", + "context": ["water", "unspecified"], }, - field_mapping["target"], + transformations, ) actual = s.unit.conversion_factor(t.unit) - assert math.isnan(actual) + assert math.isnan(actual), f"Expected actual to be NaN, but got {actual}" -def test_get_conversion_factor_m3y(field_mapping): +def test_get_conversion_factor_m3y(transformations): s = Flow( { "name": "Volume occupied, reservoir", "unit": "m3y", - "categories": ["Resources", "in water"], + "context": ["Resources", "in water"], }, - field_mapping["source"], + transformations, ) t = Flow( { - "@id": "9a9d71c7-79f7-42d0-af47-282d22a7cf07", - "@unitId": "481b9712-c417-44f1-bfba-38d58088173c", - "name": {"@xml:lang": "en", "#text": "Volume occupied, reservoir"}, - "unitName": {"@xml:lang": "en", "#text": "m3*year"}, - "compartment": { - "@subcompartmentId": "30347aef-a90b-46ba-8746-b53741aa779d", - "compartment": {"@xml:lang": "en", "#text": "natural resource"}, - "subcompartment": {"@xml:lang": "en", "#text": "in water"}, - }, + "identifier": "9a9d71c7-79f7-42d0-af47-282d22a7cf07", + "name": "Volume occupied, reservoir", + "unit": "m3*year", + "context": ["natural resource", "in water"], }, - field_mapping["target"], + transformations, ) actual = s.unit.conversion_factor(t.unit) expected = 1 - assert actual == expected + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" -def test_get_conversion_factor_m2a(field_mapping): +def test_get_conversion_factor_m2a(transformations): s = Flow( { "name": "Occupation, annual crop", "unit": "m2a", - "categories": ["Resources", "land"], + "context": ["Resources", "land"], }, - field_mapping["source"], + transformations, ) t = Flow( { - "@id": "c5aafa60-495c-461c-a1d4-b262a34c45b9", - "@unitId": "eb955b7c-7bed-401f-9c76-5db716ca3640", - "name": {"@xml:lang": "en", "#text": "Occupation, annual crop"}, - "unitName": {"@xml:lang": "en", "#text": "m2*year"}, - "compartment": { - "@subcompartmentId": "7d704b6f-d455-4f41-9c28-50b4f372f315", - "compartment": {"@xml:lang": "en", "#text": "natural resource"}, - "subcompartment": {"@xml:lang": "en", "#text": "land"}, - }, + "identifier": "c5aafa60-495c-461c-a1d4-b262a34c45b9", + "name": "Occupation, annual crop", + "unit": "m2*year", + "context": ["natural resource", "land"], }, - field_mapping["target"], + transformations, ) actual = s.unit.conversion_factor(t.unit) expected = 1 - assert actual == expected + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" -def test_get_conversion_factor_nan(field_mapping): +def test_get_conversion_factor_nan(transformations): s = Flow( { "name": "Radium-226/kg", "unit": "kg", - "categories": ["Emissions to water", ""], + "context": ["Emissions to water", ""], }, - field_mapping["source"], + transformations, ) t = Flow( { - "@id": "74a0aabb-e11b-4f3b-8921-45e447b33393", - "@unitId": "4923348e-591b-4772-b224-d19df86f04c9", - "@casNumber": "013982-63-3", - "name": {"@xml:lang": "en", "#text": "Radium-226"}, - "unitName": {"@xml:lang": "en", "#text": "kBq"}, - "compartment": { - "@subcompartmentId": "65f8d2a1-63ed-479c-b86c-3bcf38e86320", - "compartment": {"@xml:lang": "en", "#text": "water"}, - "subcompartment": {"@xml:lang": "en", "#text": "ocean"}, - }, + "identifier": "74a0aabb-e11b-4f3b-8921-45e447b33393", + "cas_number": "013982-63-3", + "name": "Radium-226", + "unit": "kBq", + "context": ["water", "ocean"], }, - field_mapping["target"], + transformations, ) actual = s.unit.conversion_factor(t.unit) - assert math.isnan(actual) + assert math.isnan(actual), f"Expected actual to be NaN, but got {actual}" diff --git a/tests/test_id_generation.py b/tests/test_id_generation.py deleted file mode 100644 index cfb510c..0000000 --- a/tests/test_id_generation.py +++ /dev/null @@ -1,23 +0,0 @@ -from flowmapper.utils import generate_flow_id - -def test_generate_flow_id(): - flow1 = { - "name": "1,4-Butanediol", - "categories": ["Air", "(unspecified)"], - "unit": "kg", - "CAS": "000110-63-4", - } - - flow2 = { - "CAS": "000110-63-4", - "name": "1,4-Butanediol", - "categories": [ - "Air", - "(unspecified)", - ], - "unit": "kg", - } - - expected = '9b680160081438f81d551c724c30b09b' - assert generate_flow_id(flow1) == expected - assert generate_flow_id(flow2) == expected diff --git a/tests/test_jsonpath.py b/tests/test_jsonpath.py deleted file mode 100644 index a474944..0000000 --- a/tests/test_jsonpath.py +++ /dev/null @@ -1,48 +0,0 @@ -import flowmapper.jsonpath as jp - -def test_basic_access(): - data = { - "name": "Chrysotile", - "context": ["Resources", "in ground"], - "unit": {"kg": 1}, - } - - assert ( - jp.extract( - "name", - data, - ) - == "Chrysotile" - ) - assert jp.extract("context.0", data) == "Resources" - - -def test_special_characters(): - data = { - "@id": "be73218b-18af-492e-96e6-addd309d1e32", - "name": {"#text": "Zinc, in ground", "@xml:lang": "en"}, - } - - assert jp.extract("@id", data) == "be73218b-18af-492e-96e6-addd309d1e32" - assert jp.extract("name.#text", data) == "Zinc, in ground" - - -def test_nested_data_access(): - data = { - "@id": "f0cc0453-32c0-48f5-b8d4-fc87d100b8d9", - "@casNumber": "000078-79-5", - "name": {"@xml:lang": "en", "#text": "Isoprene"}, - "synonym": [ - {"@xml:lang": "en", "#text": "2-methylbuta-1,3-diene"}, - {"@xml:lang": "en", "#text": "methyl bivinyl"}, - ], - } - - assert jp.extract(("synonym", ["#text"]), data) == ["2-methylbuta-1,3-diene", "methyl bivinyl"] - -def test_root(): - assert jp.root('.unit') == '' # this could be a problem if there are . in column names - assert jp.root('unit') == 'unit' - assert jp.root('unit.name') == 'unit' - assert jp.root('unit.foo.bar') == 'unit' - assert jp.root(("synonym", ["#text"])) == 'synonym' diff --git a/tests/test_match_biogenic_to_non_fossil.py b/tests/test_match_biogenic_to_non_fossil.py index 40fbf7a..4bffc53 100644 --- a/tests/test_match_biogenic_to_non_fossil.py +++ b/tests/test_match_biogenic_to_non_fossil.py @@ -1,13 +1,11 @@ -from flowmapper.flow import Flow +from flowmapper.domain import Flow from flowmapper.match import match_biogenic_to_non_fossil + def test_match_biogenic_to_non_fossil(): - fields = {"name": "name", "context": "context", "unit": "unit"} - s = Flow({"name": "Oils, biogenic", "context": "air", "unit": "kg"}, fields) - t = Flow({"name": "Oils, non-fossil", "context": "air", "unit": "kg"}, fields) + s = Flow({"name": "Oils, biogenic", "context": "air", "unit": "kg"}) + t = Flow({"name": "Oils, non-fossil", "context": "air", "unit": "kg"}) - actual = match_biogenic_to_non_fossil(s, t) - expected = { - "comment": "Biogenic to non-fossil if no better match" - } - assert actual == expected + actual = match_biogenic_to_non_fossil(s, t, [], []) + expected = {"comment": "Biogenic to non-fossil if no better match"} + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" diff --git a/tests/test_match_custom_names_with_location_codes.py b/tests/test_match_custom_names_with_location_codes.py new file mode 100644 index 0000000..50d268f --- /dev/null +++ b/tests/test_match_custom_names_with_location_codes.py @@ -0,0 +1,90 @@ +from flowmapper.domain import Flow +from flowmapper.match import match_custom_names_with_location_codes + + +def test_match_custom_names_with_location_codes_extra(): + s = Flow( + { + "name": "Water (ersatz), net cons., irrigation, HU", + "context": "air", + "unit": "kg", + } + ) + t = Flow( + {"name": "water, unspecified natural origin", "context": "air", "unit": "kg"} + ) + + actual = match_custom_names_with_location_codes(s, t, [], []) + expected = { + "comment": "Custom names with location code", + "location": "HU", + "irrigation": True, + } + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_custom_names_with_location_codes_no_extra(): + s = Flow({"name": "Water, well, HU", "context": "air", "unit": "kg"}) + t = Flow({"name": "Water, well, in ground", "context": "air", "unit": "kg"}) + + actual = match_custom_names_with_location_codes(s, t, [], []) + expected = {"comment": "Custom names with location code", "location": "HU"} + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_custom_names_with_location_codes_extra_whitespace_complicated(): + s = Flow( + { + "name": "Water (ersatz), net cons., irrigation, \t RER w/o DE+NL+NO", + "context": "air", + "unit": "kg", + } + ) + t = Flow( + {"name": "water, unspecified natural origin", "context": "air", "unit": "kg"} + ) + + actual = match_custom_names_with_location_codes(s, t, [], []) + expected = { + "comment": "Custom names with location code", + "location": "RER w/o DE+NL+NO", + "irrigation": True, + } + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_custom_names_with_location_codes_no_match(): + s = Flow({"name": "Ersatz water, RER w/o DE+NL+NO", "context": "air", "unit": "kg"}) + t = Flow( + {"name": "water, unspecified natural origin", "context": "air", "unit": "kg"} + ) + result = match_custom_names_with_location_codes(s, t, [], []) + assert ( + result is None + ), f"Expected match_custom_names_with_location_codes to return None, but got {result}" + + +def test_match_custom_names_with_location_codes_conversion(): + s = Flow({"name": "Water, well, HU", "context": "air", "unit": "kilogram"}) + t = Flow( + {"name": "Water, well, in ground", "context": "air", "unit": "cubic_meter"} + ) + + actual = match_custom_names_with_location_codes(s, t, [], []) + expected = { + "comment": "Custom names with location code", + "location": "HU", + "conversion_factor": 0.001, + } + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + s = Flow({"name": "Water, well, HU", "context": "air", "unit": "cubic_meter"}) + t = Flow({"name": "Water, well, in ground", "context": "air", "unit": "kilogram"}) + + actual = match_custom_names_with_location_codes(s, t, [], []) + expected = { + "comment": "Custom names with location code", + "location": "HU", + "conversion_factor": 1000.0, + } + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" diff --git a/tests/test_match_identical_cas_numbers.py b/tests/test_match_identical_cas_numbers.py index 35a959a..9b370fa 100644 --- a/tests/test_match_identical_cas_numbers.py +++ b/tests/test_match_identical_cas_numbers.py @@ -1,76 +1,133 @@ -from deepdiff import DeepDiff - -from flowmapper.flow import Flow +from flowmapper.domain import Flow from flowmapper.match import match_identical_cas_numbers -def test_match_identical_cas_numbers(fields): +def test_match_identical_cas_numbers(transformations): source = { - "Flowable": "1-Propanol", - "CAS No": "000071-23-8", - "Formula": "", + "name": "1-Propanol", + "cas_number": "000071-23-8", + "checmical formula": "", "Synonyms": "1-Propanol", - "Unit": "kg", + "unit": "kg", "Class": "Waterborne emissions", - "Context": "Emissions to water/groundwater", + "context": "Emissions to water/groundwater", "Flow UUID": "8C31919B-2D42-4CAD-A10E-8084CCD6BE99", "Description": "Formula: C3H8O\u007f", } target = { - "Flowable": "Propanol", - "CASNo": "000071-23-8", - "Formula": "", + "name": "Propanol", + "cas_number": "000071-23-8", + "checmical formula": "", "Synonyms": "propan-1-ol, 1-propanol, propyl alcohol, n-propanol, n-propyl alcohol", - "Unit": "kg", + "unit": "kg", + "Class": "chemical", + "ExternalReference": "", + "Preferred": "", + "context": "water/ground-", + "identifier": "85500204-9d88-40ae-9f0b-3ceba0e7a74f", + "AltUnit": "", + "Var": "", + "Second CAS": "71-31-8; 19986-23-3; 71-23-8; 64118-40-7; 4712-36-1; 142583-61-7; 71-23-8", + } + + s = Flow(source, transformations) + t = Flow(target, transformations) + + # Test with t included in all_target_flows (realistic scenario) + assert match_identical_cas_numbers( + s, t, [], [t] + ), "Expected match_identical_cas_numbers to return True for flows with identical CAS numbers, but it returned False" + + +def test_match_missing_cas_numbers(transformations): + source = { + "name": "1-Propanol", + "cas_number": "", + "checmical formula": "", + "synonyms": "1-Propanol", + "unit": "kg", + "Class": "Waterborne emissions", + "context": "Emissions to water/groundwater", + "identifier": "8C31919B-2D42-4CAD-A10E-8084CCD6BE99", + "Description": "Formula: C3H8O\u007f", + } + + target = { + "name": "Propanol", + "cas_number": "", + "checmical formula": "", + "synonyms": "propan-1-ol, 1-propanol, propyl alcohol, n-propanol, n-propyl alcohol", + "unit": "kg", "Class": "chemical", "ExternalReference": "", "Preferred": "", - "Context": "water/ground-", - "FlowUUID": "85500204-9d88-40ae-9f0b-3ceba0e7a74f", + "context": "water/ground-", + "identifier": "85500204-9d88-40ae-9f0b-3ceba0e7a74f", "AltUnit": "", "Var": "", "Second CAS": "71-31-8; 19986-23-3; 71-23-8; 64118-40-7; 4712-36-1; 142583-61-7; 71-23-8", } - s = Flow(source, fields['source']) - t = Flow(target, fields['target']) + s = Flow(source, transformations) + t = Flow(target, transformations) - match = match_identical_cas_numbers(s, t) - assert match + assert not match_identical_cas_numbers( + s, t, [], [] + ), "Expected match_identical_cas_numbers to return False for flows with missing CAS numbers, but it returned True" -def test_match_missing_cas_numbers(fields): +def test_match_identical_cas_numbers_multiple_matches(transformations): + """Test that match doesn't occur when multiple flows have same CAS and context.""" source = { - "Flowable": "1-Propanol", - "CAS No": "", - "Formula": "", + "name": "1-Propanol", + "cas_number": "000071-23-8", + "checmical formula": "", "Synonyms": "1-Propanol", - "Unit": "kg", + "unit": "kg", "Class": "Waterborne emissions", - "Context": "Emissions to water/groundwater", + "context": "Emissions to water/groundwater", "Flow UUID": "8C31919B-2D42-4CAD-A10E-8084CCD6BE99", "Description": "Formula: C3H8O\u007f", } - target = { - "Flowable": "Propanol", - "CASNo": "", - "Formula": "", + target1 = { + "name": "Propanol", + "cas_number": "000071-23-8", + "checmical formula": "", "Synonyms": "propan-1-ol, 1-propanol, propyl alcohol, n-propanol, n-propyl alcohol", - "Unit": "kg", + "unit": "kg", "Class": "chemical", "ExternalReference": "", "Preferred": "", - "Context": "water/ground-", - "FlowUUID": "85500204-9d88-40ae-9f0b-3ceba0e7a74f", + "context": "water/ground-", + "identifier": "85500204-9d88-40ae-9f0b-3ceba0e7a74f", "AltUnit": "", "Var": "", "Second CAS": "71-31-8; 19986-23-3; 71-23-8; 64118-40-7; 4712-36-1; 142583-61-7; 71-23-8", } - s = Flow(source, fields['source']) - t = Flow(target, fields['target']) + target2 = { + "name": "1-Propanol, alternative", + "cas_number": "000071-23-8", + "checmical formula": "", + "Synonyms": "propanol", + "unit": "kg", + "Class": "chemical", + "ExternalReference": "", + "Preferred": "", + "context": "water/ground-", + "identifier": "85500204-9d88-40ae-9f0b-3ceba0e7a75g", + "AltUnit": "", + "Var": "", + } + + s = Flow(source, transformations) + t1 = Flow(target1, transformations) + t2 = Flow(target2, transformations) - match = match_identical_cas_numbers(s, t) - assert not match + # Both target flows have same CAS and context as source (after transformations) + # Should not match when there are multiple flows with same CAS and context + assert not match_identical_cas_numbers( + s, t1, [], [t1, t2] + ), "Expected match_identical_cas_numbers to return False when multiple flows have same CAS and context, but it returned True" diff --git a/tests/test_match_identical_names.py b/tests/test_match_identical_names.py index 9e24fc2..c2f8697 100644 --- a/tests/test_match_identical_names.py +++ b/tests/test_match_identical_names.py @@ -1,55 +1,53 @@ -from deepdiff import DeepDiff - -from flowmapper.flow import Flow +from flowmapper.domain import Flow from flowmapper.match import match_identical_names -def test_match_identical_names(fields): +def test_match_identical_names(transformations): source = { - "Flowable": "Carbon dioxide, in air", + "name": "Carbon dioxide, in air", "CAS No": "000124-38-9", - "Unit": "kg", - "Context": "Resources/in air", + "unit": "kg", + "context": "Resources/in air", "Flow UUID": "32722990-B7D8-44A8-BC7D-EC3A89F533FF", } target = { - "Flowable": "Carbon dioxide, in air", - "CASNo": "000124-38-9", - "Unit": "kg", - "Context": "natural resource/in air", - "FlowUUID": "cc6a1abb-b123-4ca6-8f16-38209df609be", + "name": "Carbon dioxide, in air", + "cas_number": "000124-38-9", + "unit": "kg", + "context": "natural resource/in air", + "identifier": "cc6a1abb-b123-4ca6-8f16-38209df609be", } - s = Flow(source, fields['source']) - t = Flow(target, fields['target']) + s = Flow(source, transformations) + t = Flow(target, transformations) - match = match_identical_names(s, t) - assert match + match = match_identical_names(s, t, [], []) + assert ( + match + ), f"Expected match_identical_names to return a truthy value, but got {match}" -def test_match_identical_names_jsonpath(field_mapping): +def test_match_identical_names_jsonpath(transformations): source = { "name": "Carbon dioxide, in air", - "categories": ["Raw", "(unspecified)"], + "context": ["Raw", "(unspecified)"], "unit": "kg", "CAS": "000124-38-9", } target = { - "@id": "cc6a1abb-b123-4ca6-8f16-38209df609be", - "@casNumber": "000124-38-9", - "name": {"@xml:lang": "en", "#text": "Carbon dioxide, in air"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "45bb416c-a63b-429f-8754-b3f76a069c43", - "compartment": {"@xml:lang": "en", "#text": "natural resource"}, - "subcompartment": {"@xml:lang": "en", "#text": "in air"}, - }, + "identifier": "cc6a1abb-b123-4ca6-8f16-38209df609be", + "cas_number": "000124-38-9", + "name": "Carbon dioxide, in air", + "unit": "kg", + "context": ["natural resource", "in air"], } - s = Flow(source, field_mapping['source']) - t = Flow(target, field_mapping['target']) - - match = match_identical_names(s, t) - assert not match \ No newline at end of file + s = Flow(source, transformations) + t = Flow(target, transformations) + + match = match_identical_names(s, t, [], []) + assert ( + not match + ), f"Expected match_identical_names to return a falsy value, but got {match}" diff --git a/tests/test_match_identical_names_except_missing_suffix.py b/tests/test_match_identical_names_except_missing_suffix.py index b4f7598..28a06a4 100644 --- a/tests/test_match_identical_names_except_missing_suffix.py +++ b/tests/test_match_identical_names_except_missing_suffix.py @@ -1,52 +1,49 @@ -from flowmapper.flow import Flow +from flowmapper.domain import Flow from flowmapper.match import match_identical_names_except_missing_suffix -def test_match_identical_names_except_missing_suffix(fields): +def test_match_identical_names_except_missing_suffix(transformations): source = { - "Flowable": "Copper", - "CAS No": "007440-50-8", - "Unit": "kg", - "Context": "Emissions to water/groundwater", - "Flow UUID": "F277F190-A8A4-4A2D-AAF6-F6CB3772A545", + "name": "Copper", + "cas_number": "007440-50-8", + "unit": "kg", + "context": "Emissions to water/groundwater", + "identifier": "F277F190-A8A4-4A2D-AAF6-F6CB3772A545", } - target = { - "Flowable": "Copper, ion", - "CASNo": "017493-86-6", - "Unit": "kg", - "Context": "water/ground-", - "FlowUUID": "c3b659e5-35f1-408c-8cb5-b5f9b295c76e", + "name": "Copper, ion", + "cas_number": "017493-86-6", + "unit": "kg", + "context": "water/ground-", + "identifier": "c3b659e5-35f1-408c-8cb5-b5f9b295c76e", } - s = Flow(source, fields["source"]) - t = Flow(target, fields["target"]) + s = Flow(source, transformations) + t = Flow(target, transformations) - match = match_identical_names_except_missing_suffix(s, t, suffix="ion") - assert match + result = match_identical_names_except_missing_suffix(s, t, [], [], suffix="ion") + assert ( + result + ), f"Expected match_identical_names_except_missing_suffix to return a truthy value, but got {result}" -def test_match_identical_names_except_missing_suffix_different_order(field_mapping): +def test_match_identical_names_except_missing_suffix_different_order(transformations): s = Flow( - {"name": "Iron, ion", "unit": "g", "categories": ["Emissions to air", ""]}, - field_mapping["source"], + {"name": "Iron, ion", "unit": "g", "context": ["Emissions to air", ""]}, + transformations, ) - t = Flow( { - "@id": "8dba66e2-0f2e-4038-84ef-1e40b4f573a6", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@casNumber": "007439-89-6", - "name": {"@xml:lang": "en", "#text": "Iron"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "7011f0aa-f5f9-4901-8c10-884ad8296812", - "compartment": {"@xml:lang": "en", "#text": "air"}, - "subcompartment": {"@xml:lang": "en", "#text": "unspecified"}, - }, + "identifier": "8dba66e2-0f2e-4038-84ef-1e40b4f573a6", + "cas_number": "007439-89-6", + "name": "Iron", + "unit": "kg", + "context": ["air", "unspecified"], }, - field_mapping["target"], + transformations, ) - match = match_identical_names_except_missing_suffix(s, t, suffix="ion") - assert match + result = match_identical_names_except_missing_suffix(s, t, [], [], suffix="ion") + assert ( + result + ), f"Expected match_identical_names_except_missing_suffix to return a truthy value, but got {result}" diff --git a/tests/test_match_identical_names_in_synonyms.py b/tests/test_match_identical_names_in_synonyms.py index d88f0f1..c951bd8 100644 --- a/tests/test_match_identical_names_in_synonyms.py +++ b/tests/test_match_identical_names_in_synonyms.py @@ -1,37 +1,35 @@ -from flowmapper.flow import Flow +from flowmapper.domain import Flow from flowmapper.match import match_identical_names_in_synonyms -def test_match_identical_names_in_synonyms(field_mapping): + +def test_match_identical_names_in_synonyms(transformations): source = { "name": "Sulfuric acid", "unit": "kg", - "categories": ["Emissions to water", ""], + "context": ["Emissions to water", ""], } target = { - "@id": "8570c45a-8c78-4709-9b8f-fb88314d9e9d", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@formula": "H8N2O4S", - "@casNumber": "007783-20-2", - "name": {"@xml:lang": "en", "#text": "Ammonium sulfate"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "e47f0a6c-3be8-4027-9eee-de251784f708", - "compartment": {"@xml:lang": "en", "#text": "water"}, - "subcompartment": {"@xml:lang": "en", "#text": "unspecified"}, - }, - "synonym": [ - {"@xml:lang": "en", "#text": "Diammonium sulfate"}, - {"@xml:lang": "en", "#text": "Mascagnite"}, - {"@xml:lang": "en", "#text": "Sulfuric acid"}, - {"@xml:lang": "en", "#text": "Actamaster"}, - {"@xml:lang": "en", "#text": "Diammonium salt"}, - {"@xml:lang": "en", "#text": "Dolamin"}, + "identifier": "8570c45a-8c78-4709-9b8f-fb88314d9e9d", + "chemical formula": "H8N2O4S", + "cas_number": "007783-20-2", + "name": "Ammonium sulfate", + "unit": "kg", + "context": ["water", "unspecified"], + "synonyms": [ + "Diammonium sulfate", + "Mascagnite", + "Sulfuric acid", + "Actamaster", + "Diammonium salt", + "Dolamin", ], } - s = Flow(source, field_mapping["source"]) - t = Flow(target, field_mapping["target"]) + s = Flow(source, transformations) + t = Flow(target, transformations) - match = match_identical_names_in_synonyms(s, t) - assert match + result = match_identical_names_in_synonyms(s, t, [], []) + assert ( + result + ), f"Expected match_identical_names_in_synonyms to return a truthy value, but got {result}" diff --git a/tests/test_match_names_with_country_codes.py b/tests/test_match_names_with_country_codes.py index b0f1ff3..17e6724 100644 --- a/tests/test_match_names_with_country_codes.py +++ b/tests/test_match_names_with_country_codes.py @@ -1,13 +1,67 @@ -from flowmapper.flow import Flow -from flowmapper.match import match_names_with_country_codes +from flowmapper.domain import Flow +from flowmapper.match import match_names_with_location_codes + def test_match_names_with_country_codes(): - fields = {"name": "name", "context": "context", "unit": "unit"} - s = Flow({"name": "Ammonia, NL", "context": "air", "unit": "kg"}, fields) - t = Flow({"name": "Ammonia", "context": "air", "unit": "kg"}, fields) + s = Flow({"name": "Ammonia, NL", "context": "air", "unit": "kg"}) + t = Flow({"name": "Ammonia", "context": "air", "unit": "kg"}) + + actual = match_names_with_location_codes(s, t, [], []) + expected = {"comment": "Name matching with location code", "location": "NL"} + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_names_with_country_codes_extra_whitespace(): + s = Flow({"name": "Ammonia, \tNL", "context": "air", "unit": "kg"}) + t = Flow({"name": "Ammonia", "context": "air", "unit": "kg"}) + + actual = match_names_with_location_codes(s, t, [], []) + expected = {"comment": "Name matching with location code", "location": "NL"} + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_names_with_country_codes_no_match(): + s = Flow({"name": "Ammonia-NL", "context": "air", "unit": "kg"}) + t = Flow({"name": "Ammonia", "context": "air", "unit": "kg"}) + result = match_names_with_location_codes(s, t, [], []) + assert ( + result is None + ), f"Expected match_names_with_location_codes to return None, but got {result}" + + +def test_match_names_with_country_codes_complicated_location(): + s = Flow({"name": "Ammonia, RER w/o DE+NL+NO", "context": "air", "unit": "kg"}) + t = Flow({"name": "Ammonia", "context": "air", "unit": "kg"}) + + actual = match_names_with_location_codes(s, t, [], []) + expected = { + "comment": "Name matching with location code", + "location": "RER w/o DE+NL+NO", + } + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_names_with_country_codes_water_source_conversion(): + s = Flow({"name": "Water, NL", "context": "air", "unit": "kilogram"}) + t = Flow({"name": "Water", "context": "air", "unit": "cubic_meter"}) + + actual = match_names_with_location_codes(s, t, [], []) + expected = { + "comment": "Name matching with location code", + "location": "NL", + "conversion_factor": 0.001, + } + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" + + +def test_match_names_with_country_codes_water_target_conversion(): + s = Flow({"name": "Water, NL", "context": "air", "unit": "cubic_meter"}) + t = Flow({"name": "Water", "context": "air", "unit": "kilogram"}) - actual = match_names_with_country_codes(s, t) + actual = match_names_with_location_codes(s, t, [], []) expected = { - "comment": "Names with country code", "location": "NL" + "comment": "Name matching with location code", + "location": "NL", + "conversion_factor": 1000.0, } - assert actual == expected + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" diff --git a/tests/test_match_non_ionic_state.py b/tests/test_match_non_ionic_state.py index 1715e84..c7a3b41 100644 --- a/tests/test_match_non_ionic_state.py +++ b/tests/test_match_non_ionic_state.py @@ -1,36 +1,72 @@ -from flowmapper.flow import Flow +from flowmapper.domain import Flow from flowmapper.flowmap import Flowmap + def test_match_non_ionic_state(): - fields = {"name": "name", "context": "context", "unit": "unit"} s = [ - Flow({"name": "Mercury (II)", "context": "air", "unit": "kg"}, fields), - Flow({"name": "Manganese (II)", "context": "air", "unit": "kg"}, fields), + Flow({"name": "Mercury (II)", "context": "air", "unit": "kg"}), + Flow({"name": "Manganese (II)", "context": "air", "unit": "kg"}), ] t = [ - Flow({"name": "Mercury", "context": "air", "unit": "kg"}, fields), - Flow({"name": "Manganese II", "context": "air", "unit": "kg"}, fields), + Flow({"name": "Mercury", "context": "air", "unit": "kg", "identifier": "foo"}), + Flow( + { + "name": "Manganese II", + "context": "air", + "unit": "kg", + "identifier": "bar", + } + ), ] flowmap = Flowmap(s, t) - actual = flowmap.to_randonneur() + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + }, + }, + ) + actual = dp.data["update"] expected = [ - { + { "source": {"name": "Manganese (II)", "context": "air", "unit": "kg"}, "target": { - "uuid": None, + "identifier": "bar", "name": "Manganese II", "context": "air", "unit": "kg", }, - "conversion_factor": 1, + "conversion_factor": 1.0, "comment": "With/without roman numerals in parentheses", }, { "source": {"name": "Mercury (II)", "context": "air", "unit": "kg"}, - "target": {"uuid": None, "name": "Mercury", "context": "air", "unit": "kg"}, - "conversion_factor": 1, + "target": { + "identifier": "foo", + "name": "Mercury", + "context": "air", + "unit": "kg", + }, + "conversion_factor": 1.0, "comment": "Non-ionic state if no better match", - } + }, ] - assert actual == expected + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" diff --git a/tests/test_normalize_str.py b/tests/test_normalize_str.py index cb28546..6ce5cad 100644 --- a/tests/test_normalize_str.py +++ b/tests/test_normalize_str.py @@ -1,7 +1,16 @@ from flowmapper.utils import normalize_str + def test_normalize_str(): - names = ["\u0075\u0308ber", "\u0055\u0308ber", "\u00FCber", "\u00DCber", "\u00DCber ", " \u00DCber"] - actual = len({normalize_str(name) for name in names}) - expected = 1 - assert actual == expected + names = [ + "\u0075\u0308ber", + "\u0055\u0308ber", + "\u00fcber", + "\u00dcber", + "\u00dcber ", + " \u00dcber", + None, + ] + actual = {normalize_str(name) for name in names} + expected = {"über", "Über", ""} + assert actual == expected, f"Expected actual to equal {expected}, but got {actual}" diff --git a/tests/test_preferred_synonyms.py b/tests/test_preferred_synonyms.py new file mode 100644 index 0000000..c28f625 --- /dev/null +++ b/tests/test_preferred_synonyms.py @@ -0,0 +1,584 @@ +import pytest + +from flowmapper.domain import Flow +from flowmapper.preferred_synonyms import ( + has_number_pattern_at_end, + has_roman_numeral_at_end, + match_identical_names_in_preferred_synonyms, +) + + +@pytest.mark.parametrize( + "text", + [ + "Chapter I", + "Section V", + "Appendix XXI", + "Book III", + "Part IV", + "Chapter VI", + "Section VII", + "Appendix VIII", + "Appendix VIII+", + "Appendix VIII-", + "Appendix viii", + "Book IX", + "Part X", + "Chapter XI", + "Section XV", + "Appendix XX", + "Book XXX", + "Chapter II ", # Trailing space + " Chapter III ", # Leading and trailing spaces + "Chapter (I)", # With parentheses + "Section (V+)", # With parentheses and plus + "Book (III-)", # With parentheses and minus + ], +) +def test_roman_numerals_should_match(text): + """Test that valid roman numerals at the end of strings are detected.""" + assert has_roman_numeral_at_end( + text + ), f"Expected has_roman_numeral_at_end('{text}') to return True, but it returned False" + + +@pytest.mark.parametrize( + "text", + [ + "Chapter 1", + "Appendix VIII-+", + "Section A", + "Part XL", + "Chapter L", + "Appendix C", + "Chapter DC", + "Section M", + "Part MMMCMXCIX", # 3999 + "I am at the beginning", + "This ends with I but not roman", + "", + " ", + "Chapter", + ], +) +def test_non_roman_numerals_should_not_match(text): + """Test that invalid or non-roman numerals are not detected.""" + assert not has_roman_numeral_at_end( + text + ), f"Expected has_roman_numeral_at_end('{text}') to return False, but it returned True" + + +@pytest.mark.parametrize( + "text", + [ + "Substance (1+)", + "Compound (2-)", + "Element (3)", + "Chemical (5+)", + "Material (7-)", + "Substance (9)", + "Element (11)", # Multi-digit numbers are allowed + "Substance (1+) ", # Trailing space + " Compound (2-) ", # Leading and trailing spaces + "Element (123+)", # Multiple digits with plus + "Compound (456-)", # Multiple digits with minus + ], +) +def test_number_patterns_should_match(text): + """Test that valid number patterns at the end of strings are detected.""" + assert has_number_pattern_at_end( + text + ), f"Expected has_number_pattern_at_end('{text}') to return True, but it returned False" + + +@pytest.mark.parametrize( + "text", + [ + "Chemical", + "Substance 1+", # Missing parentheses + "Molecule (1+2)", + "Compound (0)", + "Chemical ()", # Empty parentheses + "Material (+)", # Just plus sign + "Substance (-)", # Just minus sign + "Element (10)", + "Substance 1-", # Missing parentheses + "Chemical (5+-)", + "Substance 1-+", # Missing parentheses + "Molecule (1+2", # Missing closing parenthesis + "Element 1+2)", # Missing opening parenthesis + "Compound (1+2", # Missing closing parenthesis + "", + " ", + "Substance (1+2) extra", # Text after pattern + "(1+) Substance", # Pattern not at end + ], +) +def test_invalid_patterns_should_not_match(text): + """Test that invalid patterns are not detected.""" + assert not has_number_pattern_at_end( + text + ), f"Expected has_number_pattern_at_end('{text}') to return False, but it returned True" + + +def test_match_when_target_has_source_name_in_synonyms_with_roman_numeral(): + """Test matching when target has source name in synonyms and target name ends with roman numeral.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", # Ends with roman numeral + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_match_when_target_has_source_name_in_synonyms_with_number_pattern(): + """Test matching when target has source name in synonyms and target name ends with number pattern.""" + source_data = { + "name": "carbon", + "context": ["air"], + "unit": "kg", + "synonyms": ["co2"], + } + target_data = { + "name": "carbon (2+)", # Ends with number pattern + "context": ["air"], + "unit": "kg", + "synonyms": ["carbon", "c"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_match_when_source_has_target_name_in_synonyms_with_roman_numeral(): + """Test matching when source has target name in synonyms and source name ends with roman numeral.""" + source_data = { + "name": "nitrogen II", # Ends with roman numeral + "context": ["air"], + "unit": "kg", + "synonyms": ["nitrogen", "n2"], + } + target_data = { + "name": "nitrogen", + "context": ["air"], + "unit": "kg", + "synonyms": ["n2"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_match_when_source_has_target_name_in_synonyms_with_number_pattern(): + """Test matching when source has target name in synonyms and source name ends with number pattern.""" + source_data = { + "name": "oxygen (1-)", # Ends with number pattern + "context": ["air"], + "unit": "kg", + "synonyms": ["oxygen", "o2"], + } + target_data = { + "name": "oxygen", + "context": ["air"], + "unit": "kg", + "synonyms": ["n2"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_no_match_when_different_contexts(): + """Test that no match occurs when contexts are different.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", + "context": ["air"], # Different context + "unit": "kg", + "synonyms": ["water", "aqua"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result is None, f"Expected result to be None, but got {result}" + + +def test_no_match_when_name_not_in_synonyms(): + """Test that no match occurs when name is not in synonyms.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", + "context": ["ground"], + "unit": "kg", + "synonyms": ["aqua", "liquid"], # "water" not in synonyms + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result is None, f"Expected result to be None, but got {result}" + + +def test_no_match_when_no_roman_numeral_or_number_pattern(): + """Test that no match occurs when name doesn't end with roman numeral or number pattern.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water", # No roman numeral or number pattern + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result is None, f"Expected result to be None, but got {result}" + + +def test_no_match_when_name_not_contained_in_other_name(): + """Test that no match occurs when one name is not contained in the other.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "different I", # "water" not contained in "different_water I" + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result is None, f"Expected result to be None, but got {result}" + + +def test_no_match_when_no_synonyms(): + """Test that no match occurs when flows have no synonyms.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": [], # No synonyms + } + target_data = { + "name": "water I", + "context": ["ground"], + "unit": "kg", + "synonyms": [], # No synonyms + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result is None, f"Expected result to be None, but got {result}" + + +def test_custom_comment(): + """Test that custom comment is returned when provided.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + + source = Flow(source_data) + target = Flow(target_data) + + custom_comment = "Custom match comment" + result = match_identical_names_in_preferred_synonyms( + source, target, [], [], custom_comment + ) + + assert result == { + "comment": custom_comment + }, f"Expected result to be {{'comment': '{custom_comment}'}}, but got {result}" + + +def test_match_with_roman_numeral_and_plus_minus(): + """Test matching with roman numerals that have + or - signs.""" + source_data = { + "name": "iron", + "context": ["ground"], + "unit": "kg", + "synonyms": ["fe"], + } + target_data = { + "name": "iron II+", # Roman numeral with plus + "context": ["ground"], + "unit": "kg", + "synonyms": ["iron", "fe"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_match_with_number_pattern_and_plus_minus(): + """Test matching with number patterns that have + or - signs.""" + source_data = { + "name": "sodium", + "context": ["ground"], + "unit": "kg", + "synonyms": ["na"], + } + target_data = { + "name": "sodium (1+)", # Number pattern with plus + "context": ["ground"], + "unit": "kg", + "synonyms": ["sodium", "na"], + } + + source = Flow(source_data) + target = Flow(target_data) + + result = match_identical_names_in_preferred_synonyms(source, target, [], []) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_no_match_when_another_target_shares_same_synonym_different_name(): + """Test that no match occurs when another target flow with a different name shares the same synonym.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", # Ends with roman numeral + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + other_target_data = { + "name": "water II", # Different name, but also has "water" in synonyms + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "h2o"], + } + + source = Flow(source_data) + target = Flow(target_data) + other_target = Flow(other_target_data) + + result = match_identical_names_in_preferred_synonyms( + source, target, [], [other_target] + ) + + assert ( + result is None + ), f"Expected result to be None when another target shares the same synonym, but got {result}" + + +def test_no_match_when_another_target_shares_same_synonym_number_pattern(): + """Test that no match occurs when another target flow with a different name shares the same synonym (number pattern case).""" + source_data = { + "name": "carbon", + "context": ["air"], + "unit": "kg", + "synonyms": ["co2"], + } + target_data = { + "name": "carbon (2+)", # Ends with number pattern + "context": ["air"], + "unit": "kg", + "synonyms": ["carbon", "c"], + } + other_target_data = { + "name": "carbon (3+)", # Different name, but also has "carbon" in synonyms + "context": ["air"], + "unit": "kg", + "synonyms": ["carbon", "co2"], + } + + source = Flow(source_data) + target = Flow(target_data) + other_target = Flow(other_target_data) + + result = match_identical_names_in_preferred_synonyms( + source, target, [], [other_target] + ) + + assert ( + result is None + ), f"Expected result to be None when another target shares the same synonym, but got {result}" + + +def test_no_match_when_another_target_shares_same_synonym_reverse_case(): + """Test that no match occurs when another target flow shares the same synonym in reverse case (source has target name in synonyms).""" + source_data = { + "name": "nitrogen II", # Ends with roman numeral + "context": ["air"], + "unit": "kg", + "synonyms": ["nitrogen", "n2"], + } + target_data = { + "name": "nitrogen", + "context": ["air"], + "unit": "kg", + "synonyms": ["n2"], + } + other_target_data = { + "name": "nitrogen III", # Different name, but also has "nitrogen" in synonyms + "context": ["air"], + "unit": "kg", + "synonyms": ["nitrogen", "n2"], + } + + source = Flow(source_data) + target = Flow(target_data) + other_target = Flow(other_target_data) + + result = match_identical_names_in_preferred_synonyms( + source, target, [], [other_target] + ) + + assert ( + result is None + ), f"Expected result to be None when another target shares the same synonym, but got {result}" + + +def test_match_when_another_target_shares_synonym_but_different_context(): + """Test that match occurs when another target flow shares the same synonym but has a different context.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", # Ends with roman numeral + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + other_target_data = { + "name": "water II", # Different name, has "water" in synonyms, but different context + "context": ["air"], # Different context + "unit": "kg", + "synonyms": ["water", "h2o"], + } + + source = Flow(source_data) + target = Flow(target_data) + other_target = Flow(other_target_data) + + result = match_identical_names_in_preferred_synonyms( + source, target, [], [other_target] + ) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" + + +def test_match_when_another_target_same_name_different_synonym(): + """Test that match occurs when another target flow has the same name but doesn't share the same synonym.""" + source_data = { + "name": "water", + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o"], + } + target_data = { + "name": "water I", # Ends with roman numeral + "context": ["ground"], + "unit": "kg", + "synonyms": ["water", "aqua"], + } + other_target_data = { + "name": "water II", # Different name, but doesn't have "water" in synonyms + "context": ["ground"], + "unit": "kg", + "synonyms": ["h2o", "liquid"], # "water" not in synonyms + } + + source = Flow(source_data) + target = Flow(target_data) + other_target = Flow(other_target_data) + + result = match_identical_names_in_preferred_synonyms( + source, target, [], [other_target] + ) + + assert result == { + "comment": "Identical preferred synonyms" + }, f"Expected result to be {{'comment': 'Identical preferred synonyms'}}, but got {result}" diff --git a/tests/test_prepare_transformations.py b/tests/test_prepare_transformations.py new file mode 100644 index 0000000..9ea7a35 --- /dev/null +++ b/tests/test_prepare_transformations.py @@ -0,0 +1,2 @@ +# TBD +# Also include pydantic stuff diff --git a/tests/test_read_field_mapping.py b/tests/test_read_field_mapping.py deleted file mode 100644 index f4ba9d6..0000000 --- a/tests/test_read_field_mapping.py +++ /dev/null @@ -1,24 +0,0 @@ -from flowmapper.utils import read_field_mapping - -def test_read_field_mapping(): - expected = { - "source": { - "uuid": "", - "name": "name", - "synonyms": "", - "context": "categories", - "unit": "unit", - "cas": "CAS", - }, - "target": { - "uuid": "@id", - "name": "name.#text", - "synonyms": ('synonym', ['#text']), - "context": "compartment.*.#text", - "unit": "unitName.#text", - "cas": "@casNumber", - }, - } - actual = read_field_mapping("tests/data/field_mapping-sp-ei.py") - assert expected == actual - diff --git a/tests/test_read_migration_files.py b/tests/test_read_migration_files.py deleted file mode 100644 index e37c2c9..0000000 --- a/tests/test_read_migration_files.py +++ /dev/null @@ -1,12 +0,0 @@ -from flowmapper.utils import read_migration_files - -def test_read_single_migration_file(snapshot): - actual = read_migration_files("tests/data/migrations.json") - assert actual == snapshot - -def test_read_multiple_migration_files(snapshot): - actual = read_migration_files( - "tests/data/migrations.json", "tests/data/transformations.json" - ) - expected = snapshot - assert actual == expected diff --git a/tests/test_repr.py b/tests/test_repr.py deleted file mode 100644 index 82ba72a..0000000 --- a/tests/test_repr.py +++ /dev/null @@ -1,82 +0,0 @@ -from flowmapper.flow import Flow -from flowmapper.flowmap import Flowmap -from pprint import pprint - - # def __repr__(self) -> str: - # # id_original = f"uuid:{self.uuid_raw_value}" if self.uuid_raw_value else f"id:{self.id}" - # # transformed = "*" if self.raw != self.transformed else "" - # original = f"{self.name_raw_value} (in {self.unit.raw_value}) <{self.context.raw_value}> [uuid:{self.uuid_raw_value}]" - # transformed = f"{self.name.value} (in {self.unit.value}) <{self.context.value}> [uuid:{self.uuid}]" - # if self.raw == self.transformed: - # return original - # else: - # return f"Original: {original}\nTransformed: {transformed}" - -fields = { - "uuid": "@id", - "name": "name.#text", - "synonyms": ("synonym", ["#text"]), - "context": "compartment.*.#text", - "unit": "unitName.#text", - "cas": "@casNumber", -} - -transformation = { - "update": [ - { - "source": {"@id": "4f777e05-70f9-4a18-a406-d8232325073f"}, - "target": {"@id": "b6b4201e-0561-5992-912f-e729fbf04e41"}, - } - ] -} - -source = { - "@id": "4f777e05-70f9-4a18-a406-d8232325073f", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@formula": "C10H13Cl2NO3", - "@casNumber": "002008-39-1", - "name": {"@xml:lang": "en", "#text": "2,4-D amines"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "be7e06e9-0bf5-462e-99dc-fe4aee383c48", - "compartment": {"@xml:lang": "en", "#text": "air"}, - "subcompartment": { - "@xml:lang": "en", - "#text": "non-urban air or from high stacks", - }, - }, - "synonym": [ - {"@xml:lang": "en", "#text": "2-(2,4-dichlorophenoxy)acetic acid"}, - {"@xml:lang": "en", "#text": "2,4-D dimethylamine salt"}, - {"@xml:lang": "en", "#text": "N-methylmethanamine"}, - ], -} - -target = { - "@id": "b6b4201e-0561-5992-912f-e729fbf04e41", - "@unitId": "487df68b-4994-4027-8fdc-a4dc298257b7", - "@formula": "C10H13Cl2NO3", - "@casNumber": "002008-39-1", - "name": {"@xml:lang": "en", "#text": "2,4-D dimethylamine salt"}, - "unitName": {"@xml:lang": "en", "#text": "kg"}, - "compartment": { - "@subcompartmentId": "be7e06e9-0bf5-462e-99dc-fe4aee383c48", - "compartment": {"@xml:lang": "en", "#text": "air"}, - "subcompartment": { - "@xml:lang": "en", - "#text": "non-urban air or from high stacks", - }, - }, -} - -s1 = Flow(source, fields) -s2 = Flow(source, fields, transformation) - -t = Flow(target, fields) - -s1 -s2 -t - -flowmap = Flowmap([s1, s2], [t]) -pprint(flowmap.mappings) \ No newline at end of file diff --git a/tests/test_rm_parentheses_roman_numerals.py b/tests/test_rm_parentheses_roman_numerals.py deleted file mode 100644 index c403033..0000000 --- a/tests/test_rm_parentheses_roman_numerals.py +++ /dev/null @@ -1,16 +0,0 @@ -from flowmapper.utils import rm_parentheses_roman_numerals, rm_roman_numerals_ionic_state - -def test_rm_parentheses_roman_numerals(): - assert rm_parentheses_roman_numerals('chromium (iii)') == 'chromium iii' - assert rm_parentheses_roman_numerals('chromium ( iii )') == 'chromium iii' - assert rm_parentheses_roman_numerals('water (evapotranspiration)') == 'water (evapotranspiration)' - assert rm_parentheses_roman_numerals('metolachlor, (s)') == 'metolachlor, (s)' - assert rm_parentheses_roman_numerals('chromium (vi)') == 'chromium vi' - assert rm_parentheses_roman_numerals('beryllium (ii)') == 'beryllium ii' - assert rm_parentheses_roman_numerals('thallium (i)') == 'thallium i' - assert rm_parentheses_roman_numerals('tin (iv) oxide') == 'tin iv oxide' - -def test_rm_roman_numerals_ionic_state(): - assert rm_roman_numerals_ionic_state('mercury (ii)') == 'mercury' - assert rm_roman_numerals_ionic_state('manganese (ii)') == 'manganese' - assert rm_roman_numerals_ionic_state('molybdenum (vi)') == 'molybdenum' diff --git a/tests/test_stringfield.py b/tests/test_stringfield.py new file mode 100644 index 0000000..7ba576e --- /dev/null +++ b/tests/test_stringfield.py @@ -0,0 +1,126 @@ +from flowmapper.string_field import StringField + + +def test_string_field_empty(): + sf = StringField(None) + assert ( + sf.original is None + ), f"Expected sf.original to be None, but got {sf.original!r}" + assert ( + sf.normalized == "" + ), f"Expected sf.normalized to be '', but got {sf.normalized!r}" + assert sf != "", "Expected sf to not equal '', but they are equal" + assert sf != "a", "Expected sf to not equal 'a', but they are equal" + assert sf != StringField( + "a" + ), "Expected sf to not equal StringField('a'), but they are equal" + assert sf is not None, "Expected sf to not be None, but it was None" + assert not sf, f"Expected sf to be falsy, but got {sf}" + assert ( + repr(sf) == "StringField with missing original value" + ), f"Expected repr(sf) to equal 'StringField with missing original value', but got {repr(sf)!r}" + + +def test_string_field_no_transformed(): + sf = StringField("A", use_lowercase=False) + assert ( + sf.original == "A" + ), f"Expected sf.original to be 'A', but got {sf.original!r}" + assert ( + sf.normalized == "A" + ), f"Expected sf.normalized to be 'A', but got {sf.normalized!r}" + assert sf == "A", "Expected sf to equal 'A', but they are not equal" + assert sf != "a", "Expected sf to not equal 'a', but they are equal" + assert sf == StringField( + "A", use_lowercase=True + ), "Expected sf to equal StringField('A', use_lowercase=True), but they are not equal" + assert sf == StringField( + "A", use_lowercase=False + ), "Expected sf to equal StringField('A', use_lowercase=False), but they are not equal" + assert sf != "B", "Expected sf to not equal 'B', but they are equal" + assert ( + not sf.use_lowercase + ), f"Expected sf.use_lowercase to be False, but got {sf.use_lowercase}" + assert sf, f"Expected sf to be truthy, but got {sf}" + assert ( + repr(sf) == "StringField: 'A' -> 'A'" + ), f"Expected repr(sf) to equal 'StringField: 'A' -> 'A'', but got {repr(sf)!r}" + + +def test_string_field_no_transformed_lowercase(): + sf = StringField("A", use_lowercase=True) + assert ( + sf.original == "A" + ), f"Expected sf.original to be 'A', but got {sf.original!r}" + assert ( + sf.normalized == "a" + ), f"Expected sf.normalized to be 'a', but got {sf.normalized!r}" + assert sf == "a", "Expected sf to equal 'a', but they are not equal" + assert sf == "A", "Expected sf to equal 'A', but they are not equal" + assert sf == StringField( + "A", use_lowercase=True + ), "Expected sf to equal StringField('A', use_lowercase=True), but they are not equal" + assert sf == StringField( + "A", use_lowercase=False + ), "Expected sf to equal StringField('A', use_lowercase=False), but they are not equal" + assert sf != "B", "Expected sf to not equal 'B', but they are equal" + assert ( + sf.use_lowercase + ), f"Expected sf.use_lowercase to be True, but got {sf.use_lowercase}" + assert sf, f"Expected sf to be truthy, but got {sf}" + assert ( + repr(sf) == "StringField: 'A' -> 'a'" + ), f"Expected repr(sf) to equal 'StringField: 'A' -> 'a'', but got {repr(sf)!r}" + + +def test_string_field_transformed(): + sf = StringField("A*", use_lowercase=False) + assert ( + sf.original == "A*" + ), f"Expected sf.original to be 'A*', but got {sf.original!r}" + assert ( + sf.normalized == "A*" + ), f"Expected sf.normalized to be 'A*', but got {sf.normalized!r}" + assert sf != "A", "Expected sf to not equal 'A', but they are equal" + assert sf != "a*", "Expected sf to not equal 'a*', but they are equal" + assert sf == "A*", "Expected sf to equal 'A*', but they are not equal" + assert sf == StringField( + "A*", use_lowercase=True + ), "Expected sf to equal StringField('A*', use_lowercase=True), but they are not equal" + assert sf == StringField( + "A*", use_lowercase=False + ), "Expected sf to equal StringField('A*', use_lowercase=False), but they are not equal" + assert sf != "B", "Expected sf to not equal 'B', but they are equal" + assert ( + not sf.use_lowercase + ), f"Expected sf.use_lowercase to be False, but got {sf.use_lowercase}" + assert sf, f"Expected sf to be truthy, but got {sf}" + assert ( + repr(sf) == "StringField: 'A*' -> 'A*'" + ), f"Expected repr(sf) to equal 'StringField: 'A*' -> 'A*'', but got {repr(sf)!r}" + + +def test_string_field_transformed_lowercase(): + sf = StringField("A*", use_lowercase=True) + assert ( + sf.original == "A*" + ), f"Expected sf.original to be 'A*', but got {sf.original!r}" + assert ( + sf.normalized == "a*" + ), f"Expected sf.normalized to be 'a*', but got {sf.normalized!r}" + assert sf == "a*", "Expected sf to equal 'a*', but they are not equal" + assert sf == "A*", "Expected sf to equal 'A*', but they are not equal" + assert sf == StringField( + "A*", use_lowercase=True + ), "Expected sf to equal StringField('A*', use_lowercase=True), but they are not equal" + assert sf == StringField( + "A*", use_lowercase=False + ), "Expected sf to equal StringField('A*', use_lowercase=False), but they are not equal" + assert sf != "B", "Expected sf to not equal 'B', but they are equal" + assert ( + sf.use_lowercase + ), f"Expected sf.use_lowercase to be True, but got {sf.use_lowercase}" + assert sf, f"Expected sf to be truthy, but got {sf}" + assert ( + repr(sf) == "StringField: 'A*' -> 'a*'" + ), f"Expected repr(sf) to equal 'StringField: 'A*' -> 'a*'', but got {repr(sf)!r}" diff --git a/tests/test_stringlist.py b/tests/test_stringlist.py new file mode 100644 index 0000000..5d32a56 --- /dev/null +++ b/tests/test_stringlist.py @@ -0,0 +1,63 @@ +from flowmapper.string_list import StringList + + +def test_string_list_empty(): + sl = StringList([]) + assert sl.data == [], f"Expected sl.data to be [], but got {sl.data}" + assert ( + list(iter(sl)) == [] + ), f"Expected list(iter(sl)) to be [], but got {list(iter(sl))}" + assert len(sl) == 0, f"Expected len(sl) to be 0, but got {len(sl)}" + assert not sl, f"Expected sl to be falsy, but got {sl}" + assert ( + repr(sl) == "StringList: Empty" + ), f"Expected repr(sl) to equal 'StringList: Empty', but got {repr(sl)!r}" + assert 1 not in sl, "Expected 1 to not be in sl, but it was" + + +def test_string_list_no_transformed(): + sl = StringList(["A", "b"]) + assert "A" in sl, "Expected 'A' to be in sl, but it was not" + assert "b" in sl, "Expected 'b' to be in sl, but it was not" + assert len(sl) == 2, f"Expected len(sl) to be 2, but got {len(sl)}" + assert sl, f"Expected sl to be truthy, but got {sl}" + expected_repr = ( + "StringList: [\"StringField: 'A' -> 'a'\", \"StringField: 'b' -> 'b'\"]" + ) + assert ( + repr(sl) == expected_repr + ), f"Expected repr(sl) to equal {expected_repr!r}, but got {repr(sl)!r}" + assert list(iter(sl)) == [ + "a", + "b", + ], f"Expected list(iter(sl)) to equal ['a', 'b'], but got {list(iter(sl))}" + assert ( + sl.data[0].original == "A" + ), f"Expected sl.data[0].original to be 'A', but got {sl.data[0].original!r}" + assert ( + sl.data[0].normalized == "a" + ), f"Expected sl.data[0].normalized to be 'a', but got {sl.data[0].normalized!r}" + + +def test_string_list_transformed(): + sl = StringList(["A", "b"], ["A*", "b"]) + assert "A*" in sl, "Expected 'A*' to be in sl, but it was not" + assert "b" in sl, "Expected 'b' to be in sl, but it was not" + assert len(sl) == 2, f"Expected len(sl) to be 2, but got {len(sl)}" + assert sl, f"Expected sl to be truthy, but got {sl}" + expected_repr = ( + "StringList: [\"StringField: 'A' -> 'a*'\", \"StringField: 'b' -> 'b'\"]" + ) + assert ( + repr(sl) == expected_repr + ), f"Expected repr(sl) to equal {expected_repr!r}, but got {repr(sl)!r}" + assert list(iter(sl)) == [ + "a*", + "b", + ], f"Expected list(iter(sl)) to equal ['a*', 'b'], but got {list(iter(sl))}" + assert ( + sl.data[0].original == "A" + ), f"Expected sl.data[0].original to be 'A', but got {sl.data[0].original!r}" + assert ( + sl.data[0].normalized == "a*" + ), f"Expected sl.data[0].normalized to be 'a*', but got {sl.data[0].normalized!r}" diff --git a/tests/test_transform_flow.py b/tests/test_transform_flow.py index 4090163..b7b4cb4 100644 --- a/tests/test_transform_flow.py +++ b/tests/test_transform_flow.py @@ -1,19 +1,175 @@ -from flowmapper.utils import read_migration_files, read_flowlist -from flowmapper.flow import Flow +import json +from pathlib import Path + +from flowmapper.domain import Flow from flowmapper.flowmap import Flowmap +from flowmapper.transformation_mapping import prepare_transformations + +DATA_DIR = Path(__file__).parent / "data" + + +def test_transform_flow_without_default_transformations(): + transformations = prepare_transformations( + [json.load(open(DATA_DIR / "transformations.json"))] + ) + source_flows = json.load(open(DATA_DIR / "sp.json")) + source_flows = [Flow(flow, transformations) for flow in source_flows] + target_flows = json.load(open(DATA_DIR / "ei-3.7.json")) + target_flows = [Flow(flow, transformations) for flow in target_flows] + + flowmap = Flowmap(source_flows, target_flows) + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + ) + actual = dp.data["update"] + + expected = [ + { + "source": { + "name": "1,4-Butanediol", + "unit": "kg", + "context": "air", + "cas_number": "110-63-4", + }, + "target": { + "name": "1,4-Butanediol", + "unit": "kg", + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "context": ["air", "unspecified"], + "cas_number": "110-63-4", + }, + "conversion_factor": 1.0, + "comment": "Identical names", + }, + { + "source": { + "name": "1,4-Butanediol", + "unit": "kg", + "context": "air/high. pop.", + "cas_number": "110-63-4", + }, + "target": { + "name": "1,4-Butanediol", + "unit": "kg", + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "context": ["air", "unspecified"], + "cas_number": "110-63-4", + }, + "conversion_factor": 1.0, + "comment": "Identical names", + }, + ] + assert ( + actual == expected + ), f"Expected actual to equal expected, but got {actual} instead of {expected}" + -def test_transform_flow_with_conversion_factor(field_mapping, snapshot): - - source_flows_path = 'tests/data/sp.json' - target_flows_path = 'tests/data/ei-3.7.json' - - transformations = read_migration_files('tests/data/transformations.json') - - source_flows = read_flowlist(source_flows_path) - source_flows = [Flow(flow, field_mapping['source'], transformations) for flow in source_flows] - target_flows = read_flowlist(target_flows_path) - target_flows = [Flow(flow, field_mapping['target']) for flow in target_flows] +def test_transform_flow_with_default_transformations(transformations): + all_transformations = transformations + prepare_transformations( + [json.load(open(DATA_DIR / "transformations.json"))] + ) + source_flows = json.load(open(DATA_DIR / "sp.json")) + source_flows = [Flow(flow, all_transformations) for flow in source_flows] + target_flows = json.load(open(DATA_DIR / "ei-3.7.json")) + target_flows = [Flow(flow, all_transformations) for flow in target_flows] flowmap = Flowmap(source_flows, target_flows) - actual = flowmap.to_randonneur() - assert actual == snapshot + dp = flowmap.to_randonneur( + source_id="test-source", + target_id="test-target", + contributors=[{"title": "Test", "roles": ["author"], "path": "test"}], + mapping_source={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "cas_number": "cas_number", + }, + }, + mapping_target={ + "expression language": "test", + "labels": { + "name": "name", + "context": "context", + "unit": "unit", + "identifier": "identifier", + "cas_number": "cas_number", + "location": "location", + }, + }, + ) + actual = dp.data["update"] + + expected = [ + { + "comment": "Identical names", + "conversion_factor": 1.0, + "source": { + "cas_number": "110-63-4", + "context": "air", + "name": "1,4-Butanediol", + "unit": "kg", + }, + "target": { + "cas_number": "110-63-4", + "context": ["air", "unspecified"], + "identifier": "09db39be-d9a6-4fc3-8d25-1f80b23e9131", + "name": "1,4-Butanediol", + "unit": "kg", + }, + }, + { + "comment": "Identical names", + "conversion_factor": 1.2142857142857142, + "source": { + "context": "air/low. pop.", + "name": "Ammonia, as N", + "unit": "kg", + }, + "target": { + "cas_number": "7664-41-7", + "context": ["air", "non-urban air or from high stacks"], + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "name": "Ammonia", + "unit": "kg", + }, + }, + { + "comment": "Name matching with location code", + "conversion_factor": 1.0, + "location": "FR", + "source": {"context": "air/low. pop.", "name": "Ammonia, FR", "unit": "kg"}, + "target": { + "cas_number": "7664-41-7", + "context": ["air", "non-urban air or from high stacks"], + "identifier": "0f440cc0-0f74-446d-99d6-8ff0e97a2444", + "name": "Ammonia", + "unit": "kg", + }, + }, + ] + + assert actual == expected diff --git a/tests/test_unit.py b/tests/test_unit.py deleted file mode 100644 index dab8a46..0000000 --- a/tests/test_unit.py +++ /dev/null @@ -1,40 +0,0 @@ -from flowmapper.unit import Unit -import math - -def test_init(): - unit = Unit.from_dict( - {"unitName": {"@xml:lang": "en", "#text": "M2A"}}, "unitName.#text" - ) - assert unit.value == "square_meter_year" - assert unit.raw_value == "M2A" - assert unit.raw_object == {"unitName": {"@xml:lang": "en", "#text": "M2A"}} - -def test_equals(): - u1 = Unit("M2A") - u2 = Unit("m2*year") - - assert u1 == u2 - -def test_equals_mass(): - u1 = Unit("kg") - u2 = Unit("kilogram") - - assert u1 == u2 - -def test_conversion_factor(): - u1 = Unit("mg") - u2 = Unit("kg") - actual = u1.conversion_factor(u2) - expected = 1e-06 - assert actual == expected - -def test_nan_conversion_factor(): - u1 = Unit("bq") - u2 = Unit("kg") - actual = u1.conversion_factor(u2) - assert math.isnan(actual) - -def test_complex_conversions(): - u1 = Unit('square_meter_year / t') - u2 = Unit('(meter ** 2 * month) / kg') - assert u1.conversion_factor(u2) == 0.012 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..828322c --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Unit tests for flowmapper using mocks.""" diff --git a/tests/unit/test_cas.py b/tests/unit/test_cas.py new file mode 100644 index 0000000..7be9a04 --- /dev/null +++ b/tests/unit/test_cas.py @@ -0,0 +1,410 @@ +"""Unit tests for CASField class.""" + +import pytest + +from flowmapper.cas import CASField + + +class TestCASFieldInitialization: + """Test CASField initialization.""" + + def test_init_with_valid_cas_string(self): + """Test initialization with valid CAS string.""" + cas = CASField("0000096-49-1") + assert ( + cas.data == "0000096-49-1" + ), f"Expected cas.data to be '0000096-49-1', but got {cas.data!r}" + from collections import UserString + + assert isinstance( + cas, UserString + ), f"Expected cas to be an instance of UserString, but got {type(cas)}" + + def test_init_with_empty_string_raises_error(self): + """Test initialization with empty string raises ValueError.""" + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + CASField("") + + def test_init_with_none_raises_error(self): + """Test initialization with None raises TypeError.""" + with pytest.raises(TypeError, match="CASField takes only `str`, but got"): + CASField(None) # type: ignore[arg-type] + + def test_init_with_integer_raises_error(self): + """Test initialization with integer raises TypeError.""" + with pytest.raises(TypeError, match="CASField takes only `str`, but got"): + CASField(96491) # type: ignore[arg-type] + + def test_init_with_userstring(self): + """Test initialization with UserString works.""" + from collections import UserString + + us = UserString("7782-40-3") + # CASField converts UserString to string before regex search, so it works + cas = CASField(us) + assert ( + cas.data == "7782-40-3" + ), f"Expected cas.data to be '7782-40-3', but got {cas.data!r}" + assert isinstance( + cas, CASField + ), f"Expected cas to be an instance of CASField, but got {type(cas)}" + + def test_init_with_two_digits_in_front(self): + """Test initialization with whitespace.""" + assert CASField( + "94-75-7" + ), "Initialization with two numbers in front section failed" + + def test_init_with_whitespace(self): + """Test initialization with whitespace.""" + cas = CASField(" 7782-40-3 ") + assert ( + cas.data == " 7782-40-3 " + ), f"Expected cas.data to preserve whitespace, but got {cas.data!r}" + + def test_inherits_from_userstring(self): + """Test that CASField inherits from UserString.""" + cas = CASField("7782-40-3") + from collections import UserString + + assert isinstance( + cas, UserString + ), f"Expected cas to be an instance of UserString, but got {type(cas)}" + # UserString is not a subclass of str + assert not isinstance( + cas, str + ), f"Expected cas to not be an instance of str (UserString is not a subclass), but got {type(cas)}" + + def test_init_with_casfield(self): + """Test initialization with another CASField object.""" + cas1 = CASField("7782-40-3") + cas2 = CASField(cas1) + assert ( + cas2.data == "7782-40-3" + ), f"Expected cas2.data to be '7782-40-3', but got {cas2.data!r}" + assert ( + cas1 == cas2 + ), f"Expected cas1 to equal cas2, but they are not equal (cas1={cas1!r}, cas2={cas2!r})" + assert ( + cas1 is not cas2 + ), "Expected cas1 and cas2 to be different instances, but they are the same instance" + assert isinstance( + cas2, CASField + ), f"Expected cas2 to be an instance of CASField, but got {type(cas2)}" + + +class TestCASFieldDigits: + """Test CASField digits property.""" + + def test_digits_with_dashes(self): + """Test digits property with dashes.""" + cas = CASField("0000096-49-1") + assert cas.digits == [ + 0, + 0, + 0, + 0, + 0, + 9, + 6, + 4, + 9, + 1, + ], f"Expected cas.digits to be [0, 0, 0, 0, 0, 9, 6, 4, 9, 1], but got {cas.digits}" + + def test_digits_without_dashes_raises_error(self): + """Test digits property without dashes raises ValueError.""" + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + CASField("0000096491") + + def test_digits_with_empty_string_raises_error(self): + """Test digits property with empty string raises ValueError.""" + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + CASField("") + + +class TestCASFieldExport: + """Test CASField export method.""" + + def test_export_with_standard_format(self): + """Test export with standard CAS format.""" + cas = CASField("7782-40-3") + assert ( + cas.export() == "7782-40-3" + ), f"Expected cas.export() to be '7782-40-3', but got {cas.export()!r}" + + def test_export_without_dashes_raises_error(self): + """Test export without dashes raises ValueError.""" + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + CASField("7782403") + + def test_export_with_leading_zeros(self): + """Test export with leading zeros.""" + cas = CASField("0007782-40-3") + # Export keeps leading zeros in the first part + assert ( + cas.export() == "0007782-40-3" + ), f"Expected cas.export() to be '0007782-40-3', but got {cas.export()!r}" + + def test_export_with_empty_string_raises_error(self): + """Test export with empty string raises ValueError.""" + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + CASField("") + + def test_export_with_single_digit_raises_error(self): + """Test export with single digit raises ValueError.""" + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + CASField("1") + + +class TestCASFieldCheckDigitExpected: + """Test CASField check_digit_expected property.""" + + def test_check_digit_expected_valid_cas(self): + """Test check_digit_expected with CAS number.""" + cas = CASField("7732-18-5") + expected = cas.check_digit_expected + assert ( + expected == 5 + ), f"Expected check_digit_expected to be 5, but got {expected}" + + def test_check_digit_expected_invalid_cas(self): + """Test check_digit_expected with invalid CAS number.""" + cas = CASField("7782-40-2") + # Check digit is 2, but expected is 3 + expected = cas.check_digit_expected + assert ( + expected == 3 + ), f"Expected check_digit_expected to be 3, but got {expected}" + + +class TestCASFieldValid: + """Test CASField valid method.""" + + def test_valid_with_invalid_cas(self): + """Test valid with invalid CAS number.""" + cas = CASField("7782-40-2") + assert ( + not cas.valid() + ), f"Expected cas.valid() to be False, but got {cas.valid()}" + + def test_valid_with_leading_zeros(self): + """Test valid with leading zeros.""" + cas = CASField("0000096-49-1") + # Check digit calculation includes leading zeros + is_valid = cas.valid() + assert is_valid and isinstance( + is_valid, bool + ), f"Expected cas.valid() to return a bool, but got {type(is_valid)}" + + +class TestCASFieldFromString: + """Test CASField from_string method.""" + + def test_from_string_with_valid_cas(self): + """Test from_string with valid CAS number.""" + cas = CASField("7782-40-3") + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + cas.from_string("000009-49-1") + + def test_from_string_with_whitespace(self): + """Test from_string with whitespace.""" + cas = CASField("7782-40-3") + result = cas.from_string(" 7782-40-3 ") + # Testing actual behavior + assert result is None or isinstance( + result, CASField + ), f"Expected result to be None or CASField, but got {type(result)}" + + def test_from_string_with_invalid_cas(self): + """Test from_string with invalid CAS number.""" + cas = CASField("7782-40-3") + result = cas.from_string("7782-40-2") + # Invalid CAS should return None + assert ( + result is None + ), f"Expected from_string to return None for invalid CAS, but got {result}" + + def test_from_string_with_empty_string(self): + """Test from_string with empty string.""" + cas = CASField("7782-40-3") + # Empty string will fail validation in __init__ + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + cas.from_string("") + + def test_from_string_with_none(self): + """Test from_string with None.""" + cas = CASField("7782-40-3") + result = cas.from_string(None) + assert ( + result is None + ), f"Expected from_string to return None for None, but got {result}" + + def test_from_string_returns_new_instance(self): + """Test that from_string returns a new instance when valid.""" + cas = CASField("7782-40-3") + result = cas.from_string("7440-05-3") + if result is not None: + assert ( + result is not cas + ), "Expected from_string() to return a new instance, but it returned the same instance" + assert ( + cas.data == "7782-40-3" + ), f"Expected original cas.data to remain '7782-40-3', but got {cas.data!r}" + + +class TestCASFieldEquality: + """Test CASField equality comparison.""" + + def test_eq_with_same_casfield(self): + """Test equality with same CASField instance (exact data match).""" + cas1 = CASField("7440-05-3") + cas2 = CASField("7440-05-3") + # CASField to CASField comparison uses exact data comparison + assert ( + cas1 == cas2 + ), f"Expected cas1 to equal cas2, but they are not equal (cas1={cas1!r}, cas2={cas2!r})" + + def test_eq_with_different_casfield_data(self): + """Test equality with CASField having different data.""" + cas1 = CASField("7440-05-3") + cas2 = CASField("7782-40-3") + assert ( + cas1 != cas2 + ), f"Expected cas1 to not equal cas2, but they are equal (cas1={cas1!r}, cas2={cas2!r})" + + def test_eq_with_casfield_different_formatting(self): + """Test equality with CASField having same CAS but different formatting.""" + cas1 = CASField("7440-05-3") + cas2 = CASField("0007440-05-3") + # CASField to CASField uses exact data comparison, so formatting matters + assert ( + cas1 != cas2 + ), f"Expected cas1 to not equal cas2 (different formatting), but they are equal (cas1={cas1!r}, cas2={cas2!r})" + + def test_eq_with_string_exact_match(self): + """Test equality with string that exactly matches.""" + cas = CASField("7440-05-3") + assert ( + cas == "7440-05-3" + ), f"Expected cas to equal '7440-05-3', but they are not equal (cas={cas!r})" + + def test_eq_with_string_different_cas(self): + """Test equality with string containing different CAS number.""" + cas = CASField("7440-05-3") + assert ( + cas != "7782-40-3" + ), f"Expected cas to not equal '7782-40-3', but they are equal (cas={cas!r})" + + def test_eq_with_string_leading_zeros(self): + """Test equality with string containing leading zeros (should normalize).""" + cas = CASField("7440-05-3") + # String comparison uses from_string which normalizes (strips leading zeros) + assert ( + cas == "0007440-05-3" + ), f"Expected cas to equal '0007440-05-3' (normalized), but they are not equal (cas={cas!r})" + + def test_eq_with_string_whitespace(self): + """Test equality with string containing whitespace (should normalize).""" + cas = CASField("7440-05-3") + # String comparison uses from_string which normalizes (strips whitespace) + assert ( + cas == " 7440-05-3 " + ), f"Expected cas to equal ' 7440-05-3 ' (normalized), but they are not equal (cas={cas!r})" + + def test_eq_with_string_leading_zeros_and_whitespace(self): + """Test equality with string containing both leading zeros and whitespace.""" + cas = CASField("7440-05-3") + # String comparison normalizes both whitespace and leading zeros + assert ( + cas == " 0007440-05-3 " + ), f"Expected cas to equal ' 0007440-05-3 ' (normalized), but they are not equal (cas={cas!r})" + + def test_eq_with_string_invalid_cas(self): + """Test equality with string containing invalid CAS number.""" + cas = CASField("7440-05-3") + # Invalid CAS strings return None from from_string, so equality is False + assert ( + cas != "7440-05-2" + ), f"Expected cas to not equal '7440-05-2' (invalid check digit), but they are equal (cas={cas!r})" + + def test_eq_with_string_empty_string(self): + """Test equality with empty string raises ValueError.""" + cas = CASField("7440-05-3") + # Empty string is invalid CAS, so from_string raises ValueError when creating CASField + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + _ = cas == "" + + def test_eq_with_userstring(self): + """Test equality with UserString.""" + from collections import UserString + + cas = CASField("7440-05-3") + us = UserString("7440-05-3") + # UserString is handled like str in __eq__, so it should normalize + assert ( + cas == us + ), f"Expected cas to equal UserString('7440-05-3'), but they are not equal (cas={cas!r})" + + def test_eq_with_userstring_leading_zeros(self): + """Test equality with UserString containing leading zeros.""" + from collections import UserString + + cas = CASField("7440-05-3") + us = UserString("0007440-05-3") + # UserString should normalize like str + assert ( + cas == us + ), f"Expected cas to equal UserString('0007440-05-3') (normalized), but they are not equal (cas={cas!r})" + + def test_eq_with_other_types(self): + """Test equality with other types returns False.""" + cas = CASField("7440-05-3") + # Non-string, non-CASField types should return False + assert ( + cas != 744053 + ), f"Expected cas to not equal integer, but they are equal (cas={cas!r})" + assert ( + cas != None + ), f"Expected cas to not equal None, but they are equal (cas={cas!r})" + assert ( + cas != [] + ), f"Expected cas to not equal list, but they are equal (cas={cas!r})" + + def test_ne_with_different_casfield(self): + """Test inequality with different CASField.""" + cas1 = CASField("7440-05-3") + cas2 = CASField("7782-40-3") + assert ( + cas1 != cas2 + ), f"Expected cas1 to not equal cas2, but they are equal (cas1={cas1!r}, cas2={cas2!r})" + + def test_ne_with_string(self): + """Test inequality with different string.""" + cas = CASField("7440-05-3") + assert ( + cas != "7782-40-3" + ), f"Expected cas to not equal '7782-40-3', but they are equal (cas={cas!r})" + + +class TestCASFieldStringBehavior: + """Test CASField string behavior (inherited from UserString).""" + + def test_string_operations(self): + """Test that CASField behaves like a string.""" + cas = CASField("7782-40-3") + assert len(cas) == 9, f"Expected len(cas) to be 9, but got {len(cas)}" + assert ( + cas.upper() == "7782-40-3" + ), f"Expected cas.upper() to be '7782-40-3', but got {cas.upper()!r}" + assert cas.startswith( + "778" + ), f"Expected cas.startswith('778') to be True, but got {cas.startswith('778')}" + + def test_string_concatenation_raises_error(self): + """Test that CASField concatenation raises ValueError for invalid format.""" + cas1 = CASField("7782-40-3") + cas2 = CASField("7440-05-3") + # Concatenation creates a string that doesn't match CAS format, so __init__ raises ValueError + with pytest.raises(ValueError, match="Given input is not valid CAS formatting"): + _ = cas1 + " and " + cas2 diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py new file mode 100644 index 0000000..b366cba --- /dev/null +++ b/tests/unit/test_context.py @@ -0,0 +1,643 @@ +"""Unit tests for ContextField class.""" + +import pytest + +from flowmapper.context import MISSING_VALUES, ContextField + + +class TestContextFieldInitialization: + """Test ContextField initialization.""" + + def test_init_with_string(self): + """Test initialization with string.""" + c = ContextField("Raw/(unspecified)") + assert ( + c.value == "Raw/(unspecified)" + ), f"Expected c.value to be 'Raw/(unspecified)', but got {c.value!r}" + assert isinstance( + c.value, str + ), f"Expected c.value to be a str, but got {type(c.value)}" + + def test_init_with_list(self): + """Test initialization with list.""" + c = ContextField(["Raw", "(unspecified)"]) + assert c.value == [ + "Raw", + "(unspecified)", + ], f"Expected c.value to be ['Raw', '(unspecified)'], but got {c.value!r}" + assert isinstance( + c.value, list + ), f"Expected c.value to be a list, but got {type(c.value)}" + + def test_init_with_tuple(self): + """Test initialization with tuple.""" + c = ContextField(("Raw",)) + assert c.value == ( + "Raw", + ), f"Expected c.value to be ('Raw',), but got {c.value!r}" + assert isinstance( + c.value, tuple + ), f"Expected c.value to be a tuple, but got {type(c.value)}" + + def test_init_with_empty_string(self): + """Test initialization with empty string.""" + c = ContextField("") + assert c.value == "", f"Expected c.value to be '', but got {c.value!r}" + + def test_init_with_empty_list(self): + """Test initialization with empty list.""" + c = ContextField([]) + assert c.value == [], f"Expected c.value to be [], but got {c.value!r}" + + def test_init_with_empty_tuple(self): + """Test initialization with empty tuple.""" + c = ContextField(tuple([])) + assert c.value == (), f"Expected c.value to be (), but got {c.value!r}" + + +class TestContextFieldNormalize: + """Test ContextField normalize method.""" + + def test_normalize_with_string(self): + """Test normalize with string value.""" + c = ContextField("A/B") + normalized = c.normalize() + assert normalized.value == ( + "a", + "b", + ), f"Expected normalized.value to be ('a', 'b'), but got {normalized.value!r}" + assert isinstance( + normalized.value, tuple + ), f"Expected normalized.value to be a tuple, but got {type(normalized.value)}" + assert ( + c.value == "A/B" + ), f"Expected original c.value to remain 'A/B', but got {c.value!r}" + + def test_normalize_with_string_no_slash(self): + """Test normalize with string without slash.""" + c = ContextField("A-B") + normalized = c.normalize() + assert normalized.value == ( + "a-b", + ), f"Expected normalized.value to be ('a-b',), but got {normalized.value!r}" + + def test_normalize_with_list(self): + """Test normalize with list value.""" + c = ContextField(["Raw", "(unspecified)"]) + normalized = c.normalize() + assert normalized.value == ( + "raw", + ), f"Expected normalized.value to be ('raw',), but got {normalized.value!r}" + + def test_normalize_with_only_unspecified(self): + """Test normalize with only unspecified value.""" + # When the only value is unspecified, it should be kept + c = ContextField(["unspecified"]) + normalized = c.normalize() + assert normalized.value == ( + "unspecified", + ), f"Expected normalized.value to be ('unspecified',), but got {normalized.value!r}" + + # Test with (unspecified) in parentheses + c2 = ContextField(["(unspecified)"]) + normalized2 = c2.normalize() + assert normalized2.value == ( + "(unspecified)", + ), f"Expected normalized.value to be ('(unspecified)',), but got {normalized2.value!r}" + + # Test with string "unspecified" + c3 = ContextField("unspecified") + normalized3 = c3.normalize() + assert normalized3.value == ( + "unspecified", + ), f"Expected normalized.value to be ('unspecified',), but got {normalized3.value!r}" + + # Test with multipleunspecified in parentheses + c2 = ContextField(["(unspecified)", "(unspecified)"]) + normalized2 = c2.normalize() + assert normalized2.value == ( + "(unspecified)", + ), f"Expected normalized.value to be ('(unspecified)',), but got {normalized2.value!r}" + + def test_normalize_with_tuple(self): + """Test normalize with tuple value.""" + c = ContextField(("Raw",)) + normalized = c.normalize() + assert normalized.value == ( + "raw", + ), f"Expected normalized.value to be ('raw',), but got {normalized.value!r}" + + def test_normalize_with_obj_parameter(self): + """Test normalize with obj parameter.""" + c = ContextField("X/Y") + normalized = c.normalize("A/B") + assert normalized.value == ( + "a", + "b", + ), f"Expected normalized.value to be ('a', 'b'), but got {normalized.value!r}" + assert ( + c.value == "X/Y" + ), f"Expected original c.value to remain 'X/Y', but got {c.value!r}" + + def test_normalize_lowercase(self): + """Test normalize converts to lowercase.""" + c = ContextField("A-B") + normalized = c.normalize() + assert normalized.value == ( + "a-b", + ), f"Expected normalized.value to be ('a-b',), but got {normalized.value!r}" + + def test_normalize_strip(self): + """Test normalize strips whitespace.""" + c = ContextField(" A-B\t\n") + normalized = c.normalize() + assert normalized.value == ( + "a-b", + ), f"Expected normalized.value to be ('a-b',), but got {normalized.value!r}" + + def test_normalize_removes_trailing_missing_values(self): + """Test normalize removes trailing missing values.""" + c = ContextField(("A", "(unknown)")) + normalized = c.normalize() + assert normalized.value == ( + "a", + ), f"Expected normalized.value to be ('a',), but got {normalized.value!r}" + + @pytest.mark.parametrize("missing_value", MISSING_VALUES) + def test_normalize_removes_trailing_missing_value(self, missing_value): + """Test normalize removes trailing missing values.""" + c = ContextField(("A", missing_value)) + normalized = c.normalize() + assert normalized.value == ( + "a", + ), f"Expected normalized.value to be ('a',) for missing value {missing_value!r}, but got {normalized.value!r}" + + def test_normalize_removes_multiple_trailing_missing_values(self): + """Test normalize removes multiple trailing missing values.""" + c = ContextField(("A", "(unknown)", "(unspecified)")) + normalized = c.normalize() + assert normalized.value == ( + "a", + ), f"Expected normalized.value to be ('a',), but got {normalized.value!r}" + + def test_normalize_does_not_remove_leading_missing_values(self): + """Test normalize does not remove leading missing values.""" + c = ContextField(("(unknown)", "A")) + normalized = c.normalize() + assert normalized.value == ( + "(unknown)", + "a", + ), f"Expected normalized.value to be ('(unknown)', 'a'), but got {normalized.value!r}" + + def test_normalize_returns_new_instance(self): + """Test that normalize returns a new instance.""" + c = ContextField("A/B") + normalized = c.normalize() + assert ( + normalized is not c + ), "Expected normalize() to return a new instance, but it returned the same instance" + assert ( + c.value == "A/B" + ), f"Expected original c.value to remain 'A/B', but got {c.value!r}" + + def test_normalize_with_invalid_type_raises_error(self): + """Test normalize with invalid type raises ValueError.""" + + class Foo: + pass + + c = ContextField("A/B") + with pytest.raises(ValueError, match="Can't understand input context"): + c.normalize(Foo()) + + +class TestContextFieldExportAsString: + """Test ContextField export_as_string method.""" + + def test_export_as_string_with_list(self): + """Test export_as_string with list value.""" + c = ContextField(["A", "B"]) + result = c.export_as_string() + assert ( + result == "A✂️B" + ), f"Expected export_as_string() to be 'A✂️B', but got {result!r}" + + def test_export_as_string_with_tuple(self): + """Test export_as_string with tuple value.""" + c = ContextField(("A", "B")) + result = c.export_as_string() + assert ( + result == "A✂️B" + ), f"Expected export_as_string() to be 'A✂️B', but got {result!r}" + + def test_export_as_string_with_string(self): + """Test export_as_string with string value.""" + c = ContextField("A/B") + result = c.export_as_string() + assert ( + result == "A/B" + ), f"Expected export_as_string() to be 'A/B', but got {result!r}" + + def test_export_as_string_with_custom_join_character_list(self): + """Test export_as_string with custom join_character for list value.""" + c = ContextField(["A", "B"]) + result = c.export_as_string("/") + assert ( + result == "A/B" + ), f"Expected export_as_string('/') to be 'A/B', but got {result!r}" + + def test_export_as_string_with_custom_join_character_tuple(self): + """Test export_as_string with custom join_character for tuple value.""" + c = ContextField(("A", "B", "C")) + result = c.export_as_string("|") + assert ( + result == "A|B|C" + ), f"Expected export_as_string('|') to be 'A|B|C', but got {result!r}" + + def test_export_as_string_with_custom_join_character_dash(self): + """Test export_as_string with custom join_character '-'.""" + c = ContextField(["A", "B"]) + result = c.export_as_string("-") + assert ( + result == "A-B" + ), f"Expected export_as_string('-') to be 'A-B', but got {result!r}" + + def test_export_as_string_with_custom_join_character_string_value(self): + """Test export_as_string with custom join_character for string value (should not use join_character).""" + c = ContextField("A/B") + result = c.export_as_string("/") + # String values are returned as-is, join_character is not used + assert ( + result == "A/B" + ), f"Expected export_as_string('/') to be 'A/B' for string value, but got {result!r}" + + def test_export_as_string_with_custom_join_character_empty_string(self): + """Test export_as_string with custom join_character as empty string.""" + c = ContextField(["A", "B"]) + result = c.export_as_string("") + assert ( + result == "AB" + ), f"Expected export_as_string('') to be 'AB', but got {result!r}" + + def test_export_as_string_with_custom_join_character_space(self): + """Test export_as_string with custom join_character as space.""" + c = ContextField(["A", "B", "C"]) + result = c.export_as_string(" ") + assert ( + result == "A B C" + ), f"Expected export_as_string(' ') to be 'A B C', but got {result!r}" + + +class TestContextFieldEq: + """Test ContextField __eq__ method.""" + + def test_eq_with_same_contextfield(self): + """Test equality with same ContextField instance.""" + c1 = ContextField("A/B") + c2 = ContextField("A/B") + assert ( + c1 == c2 + ), f"Expected c1 to equal c2, but they are not equal (c1={c1!r}, c2={c2!r})" + + def test_eq_with_different_contextfield(self): + """Test equality with different ContextField.""" + c1 = ContextField("A/B") + c2 = ContextField("X/Y") + assert ( + c1 != c2 + ), f"Expected c1 to not equal c2, but they are equal (c1={c1!r}, c2={c2!r})" + + def test_eq_with_list_and_string(self): + """Test equality with list and string values.""" + c1 = ContextField("A/B") + c2 = ContextField(["A", "B"]) + # Different value types, so not equal + assert ( + c1 != c2 + ), f"Expected c1 to not equal c2, but they are equal (c1={c1!r}, c2={c2!r})" + + def test_eq_with_string_other(self): + """Test equality with string other.""" + c = ContextField("A/B") + # __eq__ normalizes the other value and compares + # "A/B" normalized is ('a', 'b'), but c.value is "A/B", so not equal + assert ( + c != "A/B" + ), f"Expected c to not equal 'A/B', but they are equal (c={c!r})" + + def test_eq_with_empty_contextfield(self): + """Test equality with empty ContextField.""" + c1 = ContextField("") + c2 = ContextField("") + # Empty strings are falsy, so __eq__ goes to else branch + # Empty string normalizes to ('',), so c1.value ("") != normalized c2.value (('',)) + assert ( + c1 != c2 + ), f"Expected c1 to not equal c2 for empty strings, but they are equal (c1={c1!r}, c2={c2!r})" + + def test_eq_with_other_type(self): + """Test equality with non-ContextField type.""" + c = ContextField("A/B") + assert c != 123, f"Expected c to not equal 123, but they are equal (c={c!r})" + assert c != None, f"Expected c to not equal None, but they are equal (c={c!r})" + assert c != [], f"Expected c to not equal [], but they are equal (c={c!r})" + + +class TestContextFieldBool: + """Test ContextField __bool__ method.""" + + def test_bool_with_non_empty_string(self): + """Test __bool__ with non-empty string.""" + c = ContextField("A/B") + assert bool(c) is True, f"Expected bool(c) to be True, but got {bool(c)}" + + def test_bool_with_empty_string(self): + """Test __bool__ with empty string.""" + c = ContextField("") + assert bool(c) is False, f"Expected bool(c) to be False, but got {bool(c)}" + + def test_bool_with_non_empty_list(self): + """Test __bool__ with non-empty list.""" + c = ContextField(["A", "B"]) + assert bool(c) is True, f"Expected bool(c) to be True, but got {bool(c)}" + + def test_bool_with_empty_list(self): + """Test __bool__ with empty list.""" + c = ContextField([]) + assert bool(c) is False, f"Expected bool(c) to be False, but got {bool(c)}" + + def test_bool_with_non_empty_tuple(self): + """Test __bool__ with non-empty tuple.""" + c = ContextField(("A",)) + assert bool(c) is True, f"Expected bool(c) to be True, but got {bool(c)}" + + def test_bool_with_empty_tuple(self): + """Test __bool__ with empty tuple.""" + c = ContextField(()) + assert bool(c) is False, f"Expected bool(c) to be False, but got {bool(c)}" + + +class TestContextFieldHash: + """Test ContextField __hash__ method.""" + + def test_hash_with_string(self): + """Test __hash__ with string value.""" + c = ContextField("A/B") + result = hash(c) + assert isinstance( + result, int + ), f"Expected hash(c) to be an int, but got {type(result)}" + + def test_hash_with_list_raises_error(self): + """Test __hash__ with list value raises TypeError.""" + c = ContextField(["A", "B"]) + # Lists are not hashable, so hash() raises TypeError + with pytest.raises(TypeError): + _ = hash(c) + + def test_hash_with_tuple(self): + """Test __hash__ with tuple value.""" + c = ContextField(("A", "B")) + result = hash(c) + assert isinstance( + result, int + ), f"Expected hash(c) to be an int, but got {type(result)}" + + def test_hash_same_values(self): + """Test __hash__ with same values.""" + c1 = ContextField("A/B") + c2 = ContextField("A/B") + assert hash(c1) == hash( + c2 + ), f"Expected hash(c1) to equal hash(c2), but got {hash(c1)} and {hash(c2)}" + + +class TestContextFieldIter: + """Test ContextField __iter__ method.""" + + def test_iter_with_string(self): + """Test __iter__ with string value.""" + c = ContextField("A/B") + result = list(c) + assert result == [ + "A", + "/", + "B", + ], f"Expected list(c) to be ['A', '/', 'B'], but got {result!r}" + + def test_iter_with_list(self): + """Test __iter__ with list value.""" + c = ContextField(["A", "B"]) + result = list(c) + assert result == [ + "A", + "B", + ], f"Expected list(c) to be ['A', 'B'], but got {result!r}" + + def test_iter_with_tuple(self): + """Test __iter__ with tuple value.""" + c = ContextField(("A", "B")) + result = list(c) + assert result == [ + "A", + "B", + ], f"Expected list(c) to be ['A', 'B'], but got {result!r}" + + +class TestContextFieldContains: + """Test ContextField __contains__ method.""" + + def test_contains_with_string_values(self): + """Test __contains__ with string values.""" + c1 = ContextField("A") + c2 = ContextField("A/B") + # c2 in c1 means c1 is more generic than c2 + # This checks if c1.value == c2.value[:len(c1.value)] + # "A" == "A/B"[:1] -> "A" == "A" -> True + assert ( + c2 in c1 + ), f"Expected c2 to be in c1, but it was not (c1={c1!r}, c2={c2!r})" + assert ( + c1 not in c2 + ), f"Expected c1 to not be in c2, but it was (c1={c1!r}, c2={c2!r})" + + def test_contains_with_tuple_values(self): + """Test __contains__ with tuple values.""" + c1 = ContextField(("A",)) + c2 = ContextField(("A", "B")) + # c2 in c1 means c1 is more generic than c2 + # This checks if c1.value == c2.value[:len(c1.value)] + # ("A",) == ("A", "B")[:1] -> ("A",) == ("A",) -> True + assert ( + c2 in c1 + ), f"Expected c2 to be in c1, but it was not (c1={c1!r}, c2={c2!r})" + assert ( + c1 not in c2 + ), f"Expected c1 to not be in c2, but it was (c1={c1!r}, c2={c2!r})" + + def test_contains_with_list_values(self): + """Test __contains__ with list values.""" + c1 = ContextField(["A"]) + c2 = ContextField(["A", "B"]) + # c2 in c1 means c1 is more generic than c2 + assert ( + c2 in c1 + ), f"Expected c2 to be in c1, but it was not (c1={c1!r}, c2={c2!r})" + assert ( + c1 not in c2 + ), f"Expected c1 to not be in c2, but it was (c1={c1!r}, c2={c2!r})" + + def test_contains_with_non_contextfield(self): + """Test __contains__ with non-ContextField returns False.""" + c = ContextField("A/B") + assert "A/B" not in c, f"Expected 'A/B' to not be in c, but it was (c={c!r})" + assert 123 not in c, f"Expected 123 to not be in c, but it was (c={c!r})" + + +class TestContextFieldRepr: + """Test ContextField __repr__ method.""" + + def test_repr_with_string(self): + """Test __repr__ with string value.""" + c = ContextField("A/B") + result = repr(c) + assert result == "A/B", f"Expected repr(c) to be 'A/B', but got {result!r}" + + def test_repr_with_list(self): + """Test __repr__ with list value.""" + c = ContextField(["A", "B"]) + result = repr(c) + assert ( + result == "['A', 'B']" + ), f"Expected repr(c) to be '['A', 'B']', but got {result!r}" + + def test_repr_with_tuple(self): + """Test __repr__ with tuple value.""" + c = ContextField(("A", "B")) + result = repr(c) + assert ( + result == "('A', 'B')" + ), f"Expected repr(c) to be '('A', 'B')', but got {result!r}" + + +class TestContextFieldEdgeCases: + """Test ContextField edge cases.""" + + def test_normalize_preserves_original_value(self): + """Test that normalize preserves original value.""" + c = ContextField("ORIGINAL") + normalized = c.normalize() + assert ( + c.value == "ORIGINAL" + ), f"Expected original c.value to remain 'ORIGINAL', but got {c.value!r}" + assert normalized.value == ( + "original", + ), f"Expected normalized.value to be ('original',), but got {normalized.value!r}" + + def test_multiple_normalize_calls(self): + """Test multiple normalize calls.""" + c = ContextField(" TEST ") + norm1 = c.normalize() + norm2 = norm1.normalize() + assert norm1.value == ( + "test", + ), f"Expected norm1.value to be ('test',), but got {norm1.value!r}" + assert norm2.value == ( + "test", + ), f"Expected norm2.value to be ('test',), but got {norm2.value!r}" + + def test_normalize_with_mapping_parameter(self): + """Test normalize with mapping parameter (currently not implemented).""" + c = ContextField("A/B") + # mapping parameter is accepted but not used (TODO in code) + normalized = c.normalize(mapping={"A": "X"}) + assert normalized.value == ( + "a", + "b", + ), f"Expected normalized.value to be ('a', 'b'), but got {normalized.value!r}" + + +class TestContextFieldIsResource: + """Test ContextField is_resource method.""" + + @pytest.mark.parametrize( + "value,expected", + [ + # String values that should return True (resource categories) + ("resource", True), + ("resources", True), + ("natural resource", True), + ("natural resources", True), + ("land use", True), + ("economic", True), + ("social", True), + ("raw materials", True), + ("raw", True), + # Case insensitivity + ("RESOURCE", True), + ("Natural Resource", True), + # Substring matches + ("water resource extraction", True), + ("natural resource extraction", True), + ("economic activity", True), + ("social aspect", True), + # Slash-separated strings with resource + ("resource/air", True), + # String values that should return False + ("emission", False), + ("air", False), + ("water", False), + ("", False), + ("emission/air", False), + ], + ) + def test_is_resource_with_string(self, value, expected): + """Test is_resource with string values.""" + c = ContextField(value) + assert ( + c.is_resource() is expected + ), f"Expected is_resource() to be {expected} for {value!r}, but got {c.is_resource()}" + + @pytest.mark.parametrize( + "value,expected", + [ + # List values that should return True + (["resource"], True), + (["resources"], True), + (["raw"], True), + (["land use"], True), + (["economic"], True), + (["social"], True), + (["raw materials"], True), + (["RESOURCE"], True), # Case insensitive + (["emission", "resource", "air"], True), # Multiple elements, one resource + # List values that should return False + (["emission", "air", "water"], False), + ([], False), + ], + ) + def test_is_resource_with_list(self, value, expected): + """Test is_resource with list values.""" + c = ContextField(value) + assert ( + c.is_resource() is expected + ), f"Expected is_resource() to be {expected} for {value!r}, but got {c.is_resource()}" + + @pytest.mark.parametrize( + "value,expected", + [ + # Tuple values that should return True + (("resource",), True), + (("raw",), True), + (("emission", "resource", "air"), True), # Multiple elements, one resource + # Tuple values that should return False + (("emission", "air"), False), + ((), False), + ], + ) + def test_is_resource_with_tuple(self, value, expected): + """Test is_resource with tuple values.""" + c = ContextField(value) + assert ( + c.is_resource() is expected + ), f"Expected is_resource() to be {expected} for {value!r}, but got {c.is_resource()}" diff --git a/tests/unit/test_normalized_flow.py b/tests/unit/test_normalized_flow.py new file mode 100644 index 0000000..8685267 --- /dev/null +++ b/tests/unit/test_normalized_flow.py @@ -0,0 +1,427 @@ +"""Unit tests for NormalizedFlow class.""" + +from copy import copy + +import pytest + +from flowmapper.domain import Flow, NormalizedFlow + + +class TestNormalizedFlowResetCurrent: + """Test NormalizedFlow reset_current method.""" + + def test_reset_current_resets_to_normalized(self): + """Test reset_current resets current to normalized flow.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + # Modify current + nf.update_current(name="Modified name") + assert ( + nf.current.name.data != normalized.name.data + ), "Expected current to be different from normalized after update" + + # Reset + nf.reset_current() + assert ( + nf.current.name.data == normalized.name.data + ), f"Expected current.name to equal normalized.name after reset, but got {nf.current.name.data!r} != {normalized.name.data!r}" + assert ( + nf.current.unit.data == normalized.unit.data + ), f"Expected current.unit to equal normalized.unit after reset, but got {nf.current.unit.data!r} != {normalized.unit.data!r}" + assert ( + nf.current.context.value == normalized.context.value + ), f"Expected current.context to equal normalized.context after reset, but got {nf.current.context.value!r} != {normalized.context.value!r}" + + def test_reset_current_creates_new_instance(self): + """Test reset_current creates a new Flow instance.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + # Modify current + nf.update_current(name="Modified name") + old_current_id = nf.current._id + + # Reset + nf.reset_current() + assert ( + nf.current._id != old_current_id + ), "Expected reset_current to create a new Flow instance with different _id" + assert ( + nf.current is not normalized + ), "Expected reset_current to create a copy, not reference to normalized" + + def test_reset_current_preserves_normalized(self): + """Test reset_current does not modify normalized flow.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + # Modify current multiple times + nf.update_current(name="First modification") + nf.update_current(name="Second modification") + nf.update_current(unit="g") + + # Reset + nf.reset_current() + + # Check normalized is unchanged + assert ( + normalized.name.data == "carbon dioxide" + ), f"Expected normalized.name to be unchanged, but got {normalized.name.data!r}" + # Unit is normalized (kg -> kilogram), so check normalized value + assert ( + normalized.unit.data == "kilogram" + ), f"Expected normalized.unit to be unchanged, but got {normalized.unit.data!r}" + + def test_reset_current_with_complex_flow(self): + """Test reset_current with flow containing all fields.""" + data = { + "name": "Carbon dioxide, in air", + "context": ["Raw", "(unspecified)"], + "unit": "kg", + "identifier": "test-id-123", + "location": "US", + "cas_number": "000124-38-9", + "synonyms": ["CO2"], + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + # Modify multiple fields + nf.update_current(name="Modified", unit="g", location="CA") + + # Reset + nf.reset_current() + + # Verify all fields are reset + assert ( + nf.current.name.data == normalized.name.data + ), "Expected name to be reset to normalized" + assert ( + nf.current.unit.data == normalized.unit.data + ), "Expected unit to be reset to normalized" + assert ( + nf.current.location == normalized.location + ), "Expected location to be reset to normalized" + assert ( + nf.current.identifier == normalized.identifier + ), "Expected identifier to be reset to normalized" + assert ( + nf.current.cas_number == normalized.cas_number + ), "Expected cas_number to be reset to normalized" + + +class TestNormalizedFlowUpdateCurrent: + """Test NormalizedFlow update_current method.""" + + def test_update_current_with_name(self): + """Test update_current with name parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(name="Updated name") + assert ( + nf.current.name.data == "Updated name" + ), f"Expected current.name to be 'Updated name', but got {nf.current.name.data!r}" + assert ( + nf.current.unit.data == normalized.unit.data + ), "Expected unit to remain unchanged" + assert ( + nf.current.context.value == normalized.context.value + ), "Expected context to remain unchanged" + + def test_update_current_with_unit(self): + """Test update_current with unit parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(unit="g") + assert ( + nf.current.unit.data == "g" + ), f"Expected current.unit to be 'g', but got {nf.current.unit.data!r}" + assert ( + nf.current.name.data == normalized.name.data + ), "Expected name to remain unchanged" + + def test_update_current_with_context(self): + """Test update_current with context parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(context=["water", "unspecified"]) + assert nf.current.context.value == [ + "water", + "unspecified", + ], f"Expected current.context to be ['water', 'unspecified'], but got {nf.current.context.value!r}" + + def test_update_current_with_multiple_fields(self): + """Test update_current with multiple fields.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(name="Updated name", unit="g", context="water") + assert nf.current.name.data == "Updated name", "Expected name to be updated" + assert nf.current.unit.data == "g", "Expected unit to be updated" + assert nf.current.context.value == "water", "Expected context to be updated" + + def test_update_current_with_location(self): + """Test update_current with location parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + "location": "US", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(location="CA") + assert ( + nf.current.location == "CA" + ), f"Expected current.location to be 'CA', but got {nf.current.location!r}" + + def test_update_current_with_identifier(self): + """Test update_current with identifier parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + "identifier": "original-id", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(identifier="new-id") + assert ( + nf.current.identifier == "new-id" + ), f"Expected current.identifier to be 'new-id', but got {nf.current.identifier!r}" + + def test_update_current_with_cas_number(self): + """Test update_current with cas_number parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + "cas_number": "000124-38-9", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(cas_number="000078-79-5") + # CAS numbers are normalized (leading zeros removed) when passed through from_string + assert ( + nf.current.cas_number.data == "78-79-5" + ), f"Expected current.cas_number to be '78-79-5' (normalized), but got {nf.current.cas_number.data!r}" + + def test_update_current_with_synonyms(self): + """Test update_current with synonyms parameter.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + "synonyms": ["CO2"], + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(synonyms=["CO2", "carbon dioxide"]) + assert nf.current.synonyms == [ + "CO2", + "carbon dioxide", + ], f"Expected current.synonyms to be ['CO2', 'carbon dioxide'], but got {nf.current.synonyms!r}" + + def test_update_current_creates_new_instance(self): + """Test update_current creates a new Flow instance.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + old_current_id = nf.current._id + nf.update_current(name="Updated") + assert ( + nf.current._id != old_current_id + ), "Expected update_current to create a new Flow instance with different _id" + + def test_update_current_preserves_normalized(self): + """Test update_current does not modify normalized flow.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(name="Updated", unit="g") + assert ( + normalized.name.data == "carbon dioxide" + ), "Expected normalized.name to be unchanged" + # Unit is normalized (kg -> kilogram), so check normalized value + assert ( + normalized.unit.data == "kilogram" + ), f"Expected normalized.unit to be unchanged, but got {normalized.unit.data!r}" + + def test_update_current_based_on_normalized(self): + """Test update_current uses normalized as base, not current.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + # First update + nf.update_current(name="First update") + assert nf.current.name.data == "First update", "Expected first update to work" + + # Second update - should be based on normalized, not "First update" + nf.update_current(unit="g") + assert ( + nf.current.name.data == normalized.name.data + ), "Expected name to revert to normalized value when not specified in update" + assert nf.current.unit.data == "g", "Expected unit to be updated" + + def test_update_current_with_empty_synonyms(self): + """Test update_current with empty synonyms list.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + "synonyms": ["CO2"], + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(synonyms=[]) + assert ( + nf.current.synonyms == [] + ), f"Expected current.synonyms to be empty list, but got {nf.current.synonyms!r}" + + def test_update_current_with_none_location(self): + """Test update_current with None location.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + "location": "US", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + nf.update_current(location=None) + assert ( + nf.current.location is None + ), f"Expected current.location to be None, but got {nf.current.location!r}" + + def test_update_current_with_oxidation_state(self): + """Test update_current with oxidation_state parameter.""" + data = { + "name": "Iron(II) oxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + + # Note: oxidation_state is extracted from name during normalization + # This test verifies we can update it if needed + from flowmapper.oxidation_state import OxidationState + + nf.update_current(oxidation_state=3) + assert ( + nf.current.oxidation_state.value == 3 + ), f"Expected current.oxidation_state to be 3, but got {nf.current.oxidation_state.value if nf.current.oxidation_state else None!r}" diff --git a/tests/unit/test_oxidation_state.py b/tests/unit/test_oxidation_state.py new file mode 100644 index 0000000..65e33a1 --- /dev/null +++ b/tests/unit/test_oxidation_state.py @@ -0,0 +1,433 @@ +"""Unit tests for OxidationState class.""" + +import pytest + +from flowmapper.oxidation_state import OxidationState + + +class TestOxidationStateInitialization: + """Test OxidationState initialization.""" + + def test_init_with_positive_value(self): + """Test initialization with positive value.""" + os = OxidationState(3) + assert os.value == 3, f"Expected os.value to be 3, but got {os.value}" + + def test_init_with_negative_value(self): + """Test initialization with negative value.""" + os = OxidationState(-2) + assert os.value == -2, f"Expected os.value to be -2, but got {os.value}" + + def test_init_with_zero(self): + """Test initialization with zero.""" + os = OxidationState(0) + assert os.value == 0, f"Expected os.value to be 0, but got {os.value}" + + def test_init_with_boundary_values(self): + """Test initialization with boundary values.""" + os_min = OxidationState(-5) + os_max = OxidationState(9) + assert ( + os_min.value == -5 + ), f"Expected os_min.value to be -5, but got {os_min.value}" + assert ( + os_max.value == 9 + ), f"Expected os_max.value to be 9, but got {os_max.value}" + + +class TestOxidationStateEq: + """Test OxidationState __eq__ method.""" + + def test_eq_with_same_oxidation_state(self): + """Test equality with same OxidationState instance.""" + os1 = OxidationState(3) + os2 = OxidationState(3) + assert ( + os1 == os2 + ), f"Expected os1 to equal os2, but they are not equal (os1={os1.value}, os2={os2.value})" + + def test_eq_with_different_oxidation_state(self): + """Test equality with different OxidationState.""" + os1 = OxidationState(3) + os2 = OxidationState(4) + assert ( + os1 != os2 + ), f"Expected os1 to not equal os2, but they are equal (os1={os1.value}, os2={os2.value})" + + def test_eq_with_integer(self): + """Test equality with integer.""" + os = OxidationState(3) + assert ( + os == 3 + ), f"Expected os to equal 3, but they are not equal (os={os.value})" + assert ( + os != 4 + ), f"Expected os to not equal 4, but they are equal (os={os.value})" + + def test_eq_with_negative_integer(self): + """Test equality with negative integer.""" + os = OxidationState(-2) + assert ( + os == -2 + ), f"Expected os to equal -2, but they are not equal (os={os.value})" + assert ( + os != -3 + ), f"Expected os to not equal -3, but they are equal (os={os.value})" + + def test_eq_with_zero(self): + """Test equality with zero.""" + os = OxidationState(0) + assert ( + os == 0 + ), f"Expected os to equal 0, but they are not equal (os={os.value})" + assert ( + os != 1 + ), f"Expected os to not equal 1, but they are equal (os={os.value})" + + +class TestOxidationStateHasOxidationState: + """Test OxidationState has_oxidation_state static method.""" + + def test_has_oxidation_state_with_roman_numeral_lowercase(self): + """Test has_oxidation_state with lowercase roman numeral.""" + assert OxidationState.has_oxidation_state( + "chromium (iii)" + ), "Expected has_oxidation_state('chromium (iii)') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "iron (ii)" + ), "Expected has_oxidation_state('iron (ii)') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "manganese (vi)" + ), "Expected has_oxidation_state('manganese (vi)') to return True, but it returned False" + + def test_has_oxidation_state_with_roman_numeral_uppercase(self): + """Test has_oxidation_state with uppercase roman numeral.""" + assert OxidationState.has_oxidation_state( + "Iron (II)" + ), "Expected has_oxidation_state('Iron (II)') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "Chromium (III)" + ), "Expected has_oxidation_state('Chromium (III)') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "Mercury (IV)" + ), "Expected has_oxidation_state('Mercury (IV)') to return True, but it returned False" + + def test_has_oxidation_state_with_roman_numeral_no_parentheses(self): + """Test has_oxidation_state with roman numeral without parentheses.""" + assert OxidationState.has_oxidation_state( + "chromium iii" + ), "Expected has_oxidation_state('chromium iii') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "iron II" + ), "Expected has_oxidation_state('iron II') to return True, but it returned False" + + def test_has_oxidation_state_with_number(self): + """Test has_oxidation_state with number.""" + # The new regex requires a sign before the number + assert OxidationState.has_oxidation_state( + "iron (+2)" + ), "Expected has_oxidation_state('iron (+2)') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "iron (-2)" + ), "Expected has_oxidation_state('iron (-2)') to return True, but it returned False" + # Numbers without signs or with signs after no longer match + assert not OxidationState.has_oxidation_state( + "iron (2)" + ), "Expected has_oxidation_state('iron (2)') to return False (no sign), but it returned True" + assert not OxidationState.has_oxidation_state( + "iron (3+)" + ), "Expected has_oxidation_state('iron (3+)') to return False (sign after), but it returned True" + assert not OxidationState.has_oxidation_state( + "iron (2-)" + ), "Expected has_oxidation_state('iron (2-)') to return False (sign after), but it returned True" + + def test_has_oxidation_state_with_number_no_parentheses(self): + """Test has_oxidation_state with number without parentheses.""" + # The new regex requires a sign before the number + assert OxidationState.has_oxidation_state( + "iron +3" + ), "Expected has_oxidation_state('iron +3') to return True, but it returned False" + assert OxidationState.has_oxidation_state( + "iron -2" + ), "Expected has_oxidation_state('iron -2') to return True, but it returned False" + # Numbers without signs or with signs after no longer match + assert not OxidationState.has_oxidation_state( + "iron 2" + ), "Expected has_oxidation_state('iron 2') to return False (no sign), but it returned True" + assert not OxidationState.has_oxidation_state( + "iron 2-" + ), "Expected has_oxidation_state('iron 2-') to return False (sign after), but it returned True" + assert not OxidationState.has_oxidation_state( + "iron 02-" + ), "Expected has_oxidation_state('iron 02-') to return False (sign after), but it returned True" + + def test_has_oxidation_state_without_oxidation_state(self): + """Test has_oxidation_state without oxidation state.""" + assert not OxidationState.has_oxidation_state( + "water" + ), "Expected has_oxidation_state('water') to return False, but it returned True" + assert not OxidationState.has_oxidation_state( + "iron" + ), "Expected has_oxidation_state('iron') to return False, but it returned True" + assert not OxidationState.has_oxidation_state( + "chromium oxide" + ), "Expected has_oxidation_state('chromium oxide') to return False, but it returned True" + + def test_has_oxidation_state_with_compound_identifier(self): + """Test has_oxidation_state should not match numbers in compound identifiers.""" + assert not OxidationState.has_oxidation_state( + "Ethane,, 1,1,2-trichloro-1,2,2-trifluoro-, CFC-113" + ), "Expected has_oxidation_state('Ethane,, 1,1,2-trichloro-1,2,2-trifluoro-, CFC-113') to return False, but it returned True" + + def test_has_oxidation_state_should_not_match_roman_numeral_in_word(self): + """Test has_oxidation_state should not match roman numerals embedded in words.""" + assert not OxidationState.has_oxidation_state( + "Bifenox" + ), "Expected has_oxidation_state('Bifenox') to return False, but it returned True" + + def test_has_oxidation_state_with_comma(self): + """Test has_oxidation_state with comma before oxidation state.""" + assert OxidationState.has_oxidation_state( + "iron, (II)" + ), "Expected has_oxidation_state('iron, (II)') to return True, but it returned False" + # The new regex requires a sign before the number + assert OxidationState.has_oxidation_state( + "iron, (+2)" + ), "Expected has_oxidation_state('iron, (+2)') to return True, but it returned False" + + +class TestOxidationStateFromString: + """Test OxidationState from_string class method.""" + + def test_from_string_with_roman_numeral_lowercase(self): + """Test from_string with lowercase roman numeral.""" + os, remaining = OxidationState.from_string("chromium (iii)") + assert os.value == 3, f"Expected os.value to be 3, but got {os.value}" + assert ( + remaining == "chromium" + ), f"Expected remaining to be 'chromium', but got {remaining!r}" + + def test_from_string_with_roman_numeral_uppercase(self): + """Test from_string with uppercase roman numeral.""" + os, remaining = OxidationState.from_string("Iron (II)") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "Iron" + ), f"Expected remaining to be 'Iron', but got {remaining!r}" + + def test_from_string_with_roman_numeral_no_parentheses(self): + """Test from_string with roman numeral without parentheses.""" + os, remaining = OxidationState.from_string("chromium iii") + assert os.value == 3, f"Expected os.value to be 3, but got {os.value}" + assert ( + remaining == "chromium" + ), f"Expected remaining to be 'chromium', but got {remaining!r}" + + def test_from_string_with_roman_numeral_negative(self): + """Test from_string with negative roman numeral.""" + os, remaining = OxidationState.from_string("iron (II-)") + assert os.value == -2, f"Expected os.value to be -2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_roman_numeral_positive_sign(self): + """Test from_string with positive sign in roman numeral.""" + os, remaining = OxidationState.from_string("iron (II+)") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_number(self): + """Test from_string with number.""" + # The new regex requires a sign before the number + os, remaining = OxidationState.from_string("iron (+2)") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_number_positive(self): + """Test from_string with positive number.""" + # The new regex requires a sign before the number + os, remaining = OxidationState.from_string("iron (+3)") + assert os.value == 3, f"Expected os.value to be 3, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_number_negative(self): + """Test from_string with negative number.""" + # The new regex requires a sign before the number + os, remaining = OxidationState.from_string("iron (-2)") + assert os.value == -2, f"Expected os.value to be -2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_number_no_parentheses(self): + """Test from_string with number without parentheses.""" + # The new regex requires a sign before the number + os, remaining = OxidationState.from_string("iron +2") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_number_sign_before(self): + """Test from_string with sign before number.""" + os, remaining = OxidationState.from_string("iron +3") + assert os.value == 3, f"Expected os.value to be 3, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_number_sign_before_negative(self): + """Test from_string with negative sign before number.""" + os, remaining = OxidationState.from_string("iron -2") + assert os.value == -2, f"Expected os.value to be -2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_comma(self): + """Test from_string with comma before oxidation state.""" + os, remaining = OxidationState.from_string("iron, (II)") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_with_comma_and_leading_zeros(self): + """Test from_string with comma and number with leading zeros.""" + os, remaining = OxidationState.from_string("foo, +002") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "foo" + ), f"Expected remaining to be 'foo', but got {remaining!r}" + + def test_from_string_with_whitespace(self): + """Test from_string with whitespace around oxidation state.""" + os, remaining = OxidationState.from_string("iron ( II )") + assert os.value == 2, f"Expected os.value to be 2, but got {os.value}" + assert ( + remaining == "iron" + ), f"Expected remaining to be 'iron', but got {remaining!r}" + + def test_from_string_raises_error_invalid_roman_numeral(self): + """Test from_string raises error for invalid roman numeral.""" + with pytest.raises(ValueError, match="is not a valid roman numeral"): + OxidationState.from_string("iron (IIII)") + + # Test various invalid roman numerals + invalid_cases = [ + "iron (IIII)", # Four I's in a row + "iron (VV)", # Two V's + "iron (VX)", # Invalid subtraction + ] + for invalid_case in invalid_cases: + with pytest.raises(ValueError, match="is not a valid roman numeral"): + OxidationState.from_string(invalid_case) + + def test_from_string_raises_error_both_signs(self): + """Test from_string raises error when both signs are present.""" + # The new regex only matches signs before the number, so "iron (+2-)" won't match + with pytest.raises(ValueError, match="No match found"): + OxidationState.from_string("iron (+2-)") + + def test_from_string_raises_error_no_match(self): + """Test from_string raises error when no match is found.""" + with pytest.raises(ValueError, match="No match found"): + OxidationState.from_string("iron") + with pytest.raises(ValueError, match="No match found"): + OxidationState.from_string( + "Ethane,, 1,1,2-trichloro-1,2,2-trifluoro-, CFC-113" + ) + with pytest.raises(ValueError, match="No match found"): + OxidationState.from_string("Bifenox") + + def test_from_string_raises_error_too_low(self): + """Test from_string raises error for value too low.""" + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (-6)") + + def test_from_string_raises_error_too_high(self): + """Test from_string raises error for value too high.""" + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (+10)") + + def test_from_string_raises_error_values_outside_bounds_roman(self): + """Test from_string raises error for roman numeral values outside bounds.""" + # Test values too low + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (VI-)") # -6 + + # Test values too high + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (X)") # 10 + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (XI)") # 11 + + def test_from_string_raises_error_values_outside_bounds_numbers(self): + """Test from_string raises error for number values outside bounds.""" + # Test values too low + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (-6)") + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (-10)") + + # Test values too high + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (+10)") + with pytest.raises(ValueError, match="outside physical bounds"): + OxidationState.from_string("iron (+15)") + + def test_from_string_boundary_values(self): + """Test from_string with boundary values.""" + os_min, remaining = OxidationState.from_string("iron (-5)") + assert ( + os_min.value == -5 + ), f"Expected os_min.value to be -5, but got {os_min.value}" + + # The new regex requires a sign before the number + os_max, remaining = OxidationState.from_string("iron (+9)") + assert ( + os_max.value == 9 + ), f"Expected os_max.value to be 9, but got {os_max.value}" + + def test_from_string_various_roman_numerals(self): + """Test from_string with various roman numerals.""" + test_cases = [ + ("iron (i)", 1), + ("iron (ii)", 2), + ("iron (iii)", 3), + ("iron (iv)", 4), + ("iron (v)", 5), + ("iron (vi)", 6), + ("iron (vii)", 7), + ("iron (viii)", 8), + ("iron (ix)", 9), + ] + for string, expected_value in test_cases: + os, remaining = OxidationState.from_string(string) + assert ( + os.value == expected_value + ), f"Expected os.value to be {expected_value} for '{string}', but got {os.value}" + + def test_from_string_remaining_string(self): + """Test from_string returns correct remaining string.""" + test_cases = [ + ("chromium (iii)", "chromium"), + ("iron (II)", "iron"), + ("manganese (vi)", "manganese"), + # The new regex requires a sign before the number + ("mercury (+2)", "mercury"), + ("tin (+3)", "tin"), + ("beryllium (-2)", "beryllium"), + ] + for string, expected_remaining in test_cases: + os, remaining = OxidationState.from_string(string) + assert ( + remaining == expected_remaining + ), f"Expected remaining to be {expected_remaining!r} for '{string}', but got {remaining!r}" diff --git a/tests/unit/test_remove_unit_slash.py b/tests/unit/test_remove_unit_slash.py new file mode 100644 index 0000000..08bad61 --- /dev/null +++ b/tests/unit/test_remove_unit_slash.py @@ -0,0 +1,227 @@ +"""Unit tests for remove_unit_slash function.""" + +from unittest.mock import patch + +from flowmapper.domain import Flow +from flowmapper.utils import remove_unit_slash + + +class TestRemoveUnitSlash: + """Test remove_unit_slash function.""" + + def test_no_match_returns_original_name(self): + """Test that remove_unit_slash returns original name when no match is found.""" + flow = Flow.from_dict({"name": "water", "unit": "kg", "context": "air"}) + + result = remove_unit_slash(flow) + assert result == "water", f"Expected result to be 'water', but got {result!r}" + + def test_match_at_end_removes_slash_and_unit(self): + """Test that remove_unit_slash removes /m3 or /kg when at end of string with whitespace.""" + # Test with /m3 at end with whitespace - unit is captured + flow = Flow.from_dict({"name": "water/m3 ", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + # match.end() == len(name), so removes from match.start() to end + assert result == "water", f"Expected result to be 'water', but got {result!r}" + + # Test with /kg at end with whitespace + flow = Flow.from_dict({"name": "water/kg ", "unit": "kg", "context": "air"}) + result = remove_unit_slash(flow) + assert result == "water", f"Expected result to be 'water', but got {result!r}" + + def test_match_at_end_with_comma(self): + """Test that remove_unit_slash skips match with only comma after unit at end.""" + flow = Flow.from_dict({"name": "water/m3,", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + assert ( + result == "water/m3," + ), f"Expected result to be 'water/m3,', but got {result!r}" + + def test_match_in_middle_replaces_with_comma_space(self): + """Test that remove_unit_slash replaces /m3 or /kg in middle with ', '.""" + flow = Flow.from_dict( + {"name": "water/m3, pure", "unit": "m3", "context": "air"} + ) + + result = remove_unit_slash(flow) + assert ( + result == "water, pure" + ), f"Expected result to be 'water, pure', but got {result!r}" + + # Test with /kg + flow = Flow.from_dict( + {"name": "water/kg, pure", "unit": "kg", "context": "air"} + ) + result = remove_unit_slash(flow) + assert ( + result == "water, pure" + ), f"Expected result to be 'water, pure', but got {result!r}" + + def test_match_with_whitespace(self): + """Test that remove_unit_slash handles whitespace after unit.""" + flow = Flow.from_dict({"name": "water/m3 ", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + # match.end() == len(name) (whitespace is included in match), so removes from start to end + assert result == "water", f"Expected result to be 'water', but got {result!r}" + + def test_match_with_comma_and_whitespace(self): + """Test that remove_unit_slash handles comma and whitespace.""" + flow = Flow.from_dict( + {"name": "water/m3, pure", "unit": "m3", "context": "air"} + ) + + result = remove_unit_slash(flow) + assert ( + result == "water, pure" + ), f"Expected result to be 'water, pure', but got {result!r}" + + def test_multiple_matches_skipped(self): + """Test that remove_unit_slash only processes the first match.""" + # With the fixed regex, /kg at the end will match, so it removes /kg + flow = Flow.from_dict({"name": "water/m3/kg", "unit": "kg", "context": "air"}) + + result = remove_unit_slash(flow) + # The regex matches /kg at the end, so it removes /kg + assert ( + result == "water/m3" + ), f"Expected result to be 'water/m3' (removes /kg at end), but got {result!r}" + + def test_no_match_without_slash_and_unit(self): + """Test that remove_unit_slash doesn't match strings without slash and unit.""" + # This was the original bug - "Caesium I" should not match + flow = Flow.from_dict({"name": "Caesium I", "unit": "kg", "context": "air"}) + + result = remove_unit_slash(flow) + # Should not match because there's no /m3 or /kg + assert ( + result == "Caesium I" + ), f"Expected result to be 'Caesium I' (no match), but got {result!r}" + + @patch("flowmapper.utils.logger") + def test_incompatible_unit_logs_warning(self, mock_logger): + """Test that remove_unit_slash logs warning for incompatible units.""" + # Create flow with m3 in name but kg as unit (incompatible) + flow = Flow.from_dict({"name": "water/m3 ", "unit": "kg", "context": "air"}) + + # Should still return the modified name + result = remove_unit_slash(flow) + assert result == "water", f"Expected result to be 'water', but got {result!r}" + # Verify warning was called + mock_logger.warning.assert_called_once() + + @patch("flowmapper.utils.logger") + def test_incompatible_unit_logs_warning_message(self, mock_logger): + """Test that remove_unit_slash logs the correct warning message for incompatible units.""" + # Create flow with m3 in name but kg as unit (incompatible) + flow = Flow.from_dict({"name": "water/m3 pure", "unit": "kg", "context": "air"}) + + result = remove_unit_slash(flow) + assert ( + result == "water, pure" + ), f"Expected result to be 'water, pure', but got {result!r}" + + # Verify warning was called + mock_logger.warning.assert_called_once() + warning_call = mock_logger.warning.call_args[0][0] + assert ( + "has unit" in warning_call + ), f"Expected warning message to contain 'has unit', but got {warning_call!r}" + assert ( + "but name refers to incompatible unit" in warning_call + ), f"Expected warning message to contain 'but name refers to incompatible unit', but got {warning_call!r}" + assert ( + "m3" in warning_call + ), f"Expected warning message to contain 'm3', but got {warning_call!r}" + + @patch("flowmapper.utils.logger") + def test_incompatible_unit_logs_warning_with_kg(self, mock_logger): + """Test that remove_unit_slash logs warning message with kg unit.""" + # Create flow with kg in name but m3 as unit (incompatible) + flow = Flow.from_dict({"name": "water/kg pure", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + assert ( + result == "water, pure" + ), f"Expected result to be 'water, pure', but got {result!r}" + + # Verify warning was called with kg + mock_logger.warning.assert_called_once() + warning_call = mock_logger.warning.call_args[0][0] + assert ( + "kg" in warning_call + ), f"Expected warning message to contain 'kg', but got {warning_call!r}" + + @patch("flowmapper.utils.logger") + def test_compatible_unit_no_warning(self, mock_logger): + """Test that remove_unit_slash doesn't log warning for compatible units.""" + # Create flow with m3 in name and m3 as unit (compatible) + flow = Flow.from_dict({"name": "water/m3 ", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + assert result == "water", f"Expected result to be 'water', but got {result!r}" + # Verify warning was NOT called for compatible units + mock_logger.warning.assert_not_called() + + def test_match_when_unit_not_followed_by_whitespace_or_comma(self): + """Test that remove_unit_slash doesn't match when unit is not followed by whitespace or comma.""" + flow = Flow.from_dict({"name": "water/m3x", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + # The regex requires whitespace, comma, or end of string after /m3 or /kg + # Since /m3x doesn't match, no change should occur + assert ( + result == "water/m3x" + ), f"Expected result to be 'water/m3x' (no match), but got {result!r}" + + def test_match_not_at_end_replaces(self): + """Test that remove_unit_slash replaces match when not at end.""" + flow = Flow.from_dict({"name": "water/m3 pure", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + assert ( + result == "water, pure" + ), f"Expected result to be 'water, pure', but got {result!r}" + + def test_case_sensitivity(self): + """Test that remove_unit_slash is case-sensitive for unit pattern.""" + flow = Flow.from_dict( + {"name": "water/M3", "unit": "m3", "context": "air"} + ) # Uppercase M3 + + # Should not match uppercase M3 + result = remove_unit_slash(flow) + assert ( + result == "water/M3" + ), f"Expected result to be 'water/M3' (no match), but got {result!r}" + + def test_no_unit_slash_pattern(self): + """Test that remove_unit_slash doesn't match other slash patterns.""" + flow = Flow.from_dict({"name": "water/liter", "unit": "kg", "context": "air"}) + + result = remove_unit_slash(flow) + assert ( + result == "water/liter" + ), f"Expected result to be 'water/liter' (no match), but got {result!r}" + + def test_empty_name(self): + """Test that remove_unit_slash handles empty name.""" + flow = Flow.from_dict({"name": "", "unit": "kg", "context": "air"}) + + result = remove_unit_slash(flow) + assert result == "", f"Expected result to be '', but got {result!r}" + + def test_name_with_only_unit_slash(self): + """Test that remove_unit_slash handles name with only /m3 or /kg with whitespace.""" + flow = Flow.from_dict({"name": "/m3 ", "unit": "m3", "context": "air"}) + + result = remove_unit_slash(flow) + # match.end() == len(name), so removes from match.start() to end + assert result == "", f"Expected result to be '', but got {result!r}" + + # Test with /kg + flow = Flow.from_dict({"name": "/kg ", "unit": "kg", "context": "air"}) + result = remove_unit_slash(flow) + assert result == "", f"Expected result to be '', but got {result!r}" diff --git a/tests/unit/test_split_location_suffix.py b/tests/unit/test_split_location_suffix.py new file mode 100644 index 0000000..be5e342 --- /dev/null +++ b/tests/unit/test_split_location_suffix.py @@ -0,0 +1,129 @@ +"""Unit tests for split_location_suffix function.""" + +from flowmapper.location import split_location_suffix + + +class TestSplitLocationSuffix: + """Test split_location_suffix function.""" + + def test_simple_location_code(self): + """Test split_location_suffix with simple location code.""" + name, location = split_location_suffix("Ammonia, NL") + assert name == "Ammonia", f"Expected name to be 'Ammonia', but got {name!r}" + assert location == "NL", f"Expected location to be 'NL', but got {location!r}" + + def test_location_code_with_extra_whitespace(self): + """Test split_location_suffix with extra whitespace.""" + name, location = split_location_suffix("Ammonia, \tNL") + assert name == "Ammonia", f"Expected name to be 'Ammonia', but got {name!r}" + assert location == "NL", f"Expected location to be 'NL', but got {location!r}" + + def test_complicated_location_code(self): + """Test split_location_suffix with complicated location code.""" + name, location = split_location_suffix("Ammonia, RER w/o DE+NL+NO") + assert name == "Ammonia", f"Expected name to be 'Ammonia', but got {name!r}" + assert ( + location == "RER w/o DE+NL+NO" + ), f"Expected location to be 'RER w/o DE+NL+NO', but got {location!r}" + + def test_no_location_code(self): + """Test split_location_suffix with no location code.""" + name, location = split_location_suffix("Ammonia") + assert name == "Ammonia", f"Expected name to be 'Ammonia', but got {name!r}" + assert location is None, f"Expected location to be None, but got {location!r}" + + def test_location_code_with_dash(self): + """Test split_location_suffix with location code using dash (should not match).""" + name, location = split_location_suffix("Ammonia-NL") + assert ( + name == "Ammonia-NL" + ), f"Expected name to be 'Ammonia-NL', but got {name!r}" + assert location is None, f"Expected location to be None, but got {location!r}" + + def test_location_code_case_insensitive_fails(self): + """Test split_location_suffix is case-insensitive for location codes.""" + name, location = split_location_suffix("Ammonia, nl") + assert ( + name == "Ammonia, nl" + ), f"Expected name to be 'Ammonia, nl', but got {name!r}" + assert location is None, f"Expected location to be 'None', but got {location!r}" + + def test_multiple_commas(self): + """Test split_location_suffix with multiple commas.""" + name, location = split_location_suffix("Ammonia, pure, NL") + # Should match the last comma followed by location code + assert ( + name == "Ammonia, pure" + ), f"Expected name to be 'Ammonia, pure', but got {name!r}" + assert location == "NL", f"Expected location to be 'NL', but got {location!r}" + + def test_location_code_in_middle(self): + """Test split_location_suffix with location code not at end.""" + name, location = split_location_suffix("Ammonia, NL, pure") + # Should not match because location code is not at the end + assert ( + name == "Ammonia, NL, pure" + ), f"Expected name to be 'Ammonia, NL, pure', but got {name!r}" + assert location is None, f"Expected location to be None, but got {location!r}" + + def test_empty_string(self): + """Test split_location_suffix with empty string.""" + name, location = split_location_suffix("") + assert name == "", f"Expected name to be '', but got {name!r}" + assert location is None, f"Expected location to be None, but got {location!r}" + + def test_only_location_code(self): + """Test split_location_suffix with only location code.""" + name, location = split_location_suffix(", NL") + assert name == "", f"Expected name to be '', but got {name!r}" + assert location == "NL", f"Expected location to be 'NL', but got {location!r}" + + def test_whitespace_before_comma(self): + """Test split_location_suffix with whitespace before comma.""" + name, location = split_location_suffix("Ammonia , NL") + # The regex requires comma immediately, so this might not match + # Testing actual behavior + assert ( + name == "Ammonia , NL" + ), f"Expected name to be 'Ammonia , NL' (no match), but got {name!r}" + assert location is None, f"Expected location to be None, but got {location!r}" + + def test_no_whitespace_after_comma(self): + """Test split_location_suffix with no whitespace after comma.""" + name, location = split_location_suffix("Ammonia,NL") + # The regex requires whitespace after comma + assert ( + name == "Ammonia,NL" + ), f"Expected name to be 'Ammonia,NL' (no match), but got {name!r}" + assert location is None, f"Expected location to be None, but got {location!r}" + + def test_various_location_codes(self): + """Test split_location_suffix with various location codes.""" + test_cases = [ + ("Water, DE", "Water", "DE"), + ("Water, FR", "Water", "FR"), + ("Water, US", "Water", "US"), + ("Water, GLO", "Water", "GLO"), + ] + for input_str, expected_name, expected_location in test_cases: + name, location = split_location_suffix(input_str) + assert ( + name == expected_name + ), f"Expected name to be {expected_name!r} for '{input_str}', but got {name!r}" + assert ( + location == expected_location + ), f"Expected location to be {expected_location!r} for '{input_str}', but got {location!r}" + + def test_complex_location_with_operators(self): + """Test split_location_suffix with complex location codes containing operators.""" + name, location = split_location_suffix("Ammonia, RER w/o DE+NL+NO") + assert name == "Ammonia", f"Expected name to be 'Ammonia', but got {name!r}" + assert ( + location == "RER w/o DE+NL+NO" + ), f"Expected location to be 'RER w/o DE+NL+NO', but got {location!r}" + + def test_location_code_with_trailing_whitespace(self): + """Test split_location_suffix with trailing whitespace after location.""" + name, location = split_location_suffix("Ammonia, NL ") + assert name == "Ammonia", f"Expected name to be 'Ammonia', but got {name!r}" + assert location == "NL", f"Expected location to be 'NL', but got {location!r}" diff --git a/tests/unit/test_string_field.py b/tests/unit/test_string_field.py new file mode 100644 index 0000000..3dd4536 --- /dev/null +++ b/tests/unit/test_string_field.py @@ -0,0 +1,232 @@ +"""Unit tests for StringField class.""" + +from flowmapper.string_field import StringField + + +class TestStringFieldInitialization: + """Test StringField initialization.""" + + def test_init_with_value(self): + """Test initialization with a value.""" + sf = StringField("test") + assert sf == "test", f"Expected sf to equal 'test', but got {sf!r}" + from collections import UserString + + assert isinstance( + sf, UserString + ), f"Expected sf to be an instance of UserString, but got {type(sf)}" + assert not isinstance( + sf, str + ), f"Expected sf to not be an instance of str (UserString is not a subclass), but got {type(sf)}" + + def test_init_with_empty_string(self): + """Test initialization with empty string.""" + sf = StringField("") + # Empty StringField doesn't equal empty string due to __eq__ implementation + assert sf != "", f"Expected sf to not equal '', but they are equal (sf={sf!r})" + assert sf.data == "", f"Expected sf.data to be '', but got {sf.data!r}" + + def test_init_with_whitespace(self): + """Test initialization with whitespace.""" + sf = StringField(" test ") + # Equality normalizes the other string, so " test " becomes "test" + assert sf == " test ", f"Expected sf to equal ' test ', but got {sf!r}" + assert ( + sf.data == " test " + ), f"Expected sf.data to be ' test ', but got {sf.data!r}" + + def test_inherits_from_userstring(self): + """Test that StringField inherits from UserString.""" + sf = StringField("test") + from collections import UserString + + assert isinstance( + sf, UserString + ), f"Expected sf to be an instance of UserString, but got {type(sf)}" + assert issubclass( + StringField, UserString + ), "Expected StringField to be a subclass of UserString, but it is not" + # UserString is not a subclass of str + assert not isinstance( + sf, str + ), f"Expected sf to not be an instance of str (UserString is not a subclass), but got {type(sf)}" + + +class TestStringFieldNormalize: + """Test StringField normalize method.""" + + def test_normalize_with_lowercase_default(self): + """Test normalize with default lowercase=True.""" + sf = StringField("TEST") + normalized = sf.normalize() + assert ( + normalized == "test" + ), f"Expected normalized to equal 'test', but got {normalized!r}" + assert isinstance( + normalized, StringField + ), f"Expected normalized to be a StringField instance, but got {type(normalized)}" + + def test_normalize_with_lowercase_false(self): + """Test normalize with lowercase=False.""" + sf = StringField("TEST") + normalized = sf.normalize(lowercase=False) + assert ( + normalized == "TEST" + ), f"Expected normalized to equal 'TEST', but got {normalized!r}" + + def test_normalize_with_whitespace(self): + """Test normalize with whitespace.""" + sf = StringField(" test ") + normalized = sf.normalize() + assert ( + normalized == "test" + ), f"Expected normalized to equal 'test', but got {normalized!r}" + + def test_normalize_returns_new_instance(self): + """Test that normalize returns a new instance.""" + sf = StringField("TEST") + normalized = sf.normalize() + assert ( + normalized is not sf + ), "Expected normalize() to return a new instance, but it returned the same instance" + assert sf == "TEST", f"Expected original sf to remain 'TEST', but got {sf!r}" + + +class TestStringFieldEq: + """Test StringField __eq__ method.""" + + def test_eq_with_same_stringfield(self): + """Test equality with same StringField instance.""" + sf1 = StringField("test") + sf2 = StringField("test") + assert ( + sf1 == sf2 + ), f"Expected sf1 to equal sf2, but they are not equal (sf1={sf1!r}, sf2={sf2!r})" + + def test_eq_with_different_stringfield(self): + """Test equality with different StringField.""" + sf1 = StringField("test") + sf2 = StringField("other") + assert ( + sf1 != sf2 + ), f"Expected sf1 to not equal sf2, but they are equal (sf1={sf1!r}, sf2={sf2!r})" + + def test_eq_with_string(self): + """Test equality with string.""" + sf = StringField("test") + assert ( + sf == "test" + ), f"Expected sf to equal 'test', but they are not equal (sf={sf!r})" + assert ( + sf != "other" + ), f"Expected sf to not equal 'other', but they are equal (sf={sf!r})" + + def test_eq_with_empty_stringfield(self): + """Test equality with empty StringField.""" + sf = StringField("") + assert sf != "", f"Expected sf to not equal '', but they are equal (sf={sf!r})" + assert ( + sf != "test" + ), f"Expected sf to not equal 'test', but they are equal (sf={sf!r})" + + def test_eq_with_other_type(self): + """Test equality with non-string, non-StringField type.""" + sf = StringField("test") + assert ( + sf != 123 + ), f"Expected sf to not equal 123, but they are equal (sf={sf!r})" + assert ( + sf != None + ), f"Expected sf to not equal None, but they are equal (sf={sf!r})" + assert sf != [], f"Expected sf to not equal [], but they are equal (sf={sf!r})" + + +class TestStringFieldStrBehavior: + """Test StringField string behavior (inherited from str).""" + + def test_str_operations(self): + """Test that StringField behaves like a string.""" + sf = StringField("test") + assert len(sf) == 4, f"Expected len(sf) to be 4, but got {len(sf)}" + assert ( + sf.upper() == "TEST" + ), f"Expected sf.upper() to be 'TEST', but got {sf.upper()!r}" + assert ( + sf.lower() == "test" + ), f"Expected sf.lower() to be 'test', but got {sf.lower()!r}" + assert sf.startswith( + "te" + ), f"Expected sf.startswith('te') to be True, but got {sf.startswith('te')}" + + def test_bool_with_non_empty_string(self): + """Test __bool__ with non-empty string (inherited from str).""" + sf = StringField("test") + assert bool(sf) is True, f"Expected bool(sf) to be True, but got {bool(sf)}" + + def test_bool_with_empty_string(self): + """Test __bool__ with empty string (inherited from str).""" + sf = StringField("") + assert bool(sf) is False, f"Expected bool(sf) to be False, but got {bool(sf)}" + + def test_bool_with_whitespace(self): + """Test __bool__ with whitespace-only string (inherited from str).""" + sf = StringField(" ") + assert ( + bool(sf) is True + ), f"Expected bool(sf) to be True for whitespace, but got {bool(sf)}" + + +class TestStringFieldEdgeCases: + """Test StringField edge cases.""" + + def test_value_preserved_after_normalize(self): + """Test that original value is preserved after normalize.""" + sf = StringField("ORIGINAL") + normalized = sf.normalize() + assert ( + sf == "ORIGINAL" + ), f"Expected original sf to remain 'ORIGINAL', but got {sf!r}" + assert ( + normalized == "original" + ), f"Expected normalized to be 'original', but got {normalized!r}" + + def test_multiple_normalize_calls(self): + """Test multiple normalize calls.""" + sf = StringField(" TEST ") + norm1 = sf.normalize() + norm2 = norm1.normalize() + assert norm1 == "test", f"Expected norm1 to be 'test', but got {norm1!r}" + assert norm2 == "test", f"Expected norm2 to be 'test', but got {norm2!r}" + + def test_equality_chain(self): + """Test equality chain with multiple StringFields.""" + sf1 = StringField("test") + sf2 = StringField("test") + sf3 = StringField("test") + assert ( + sf1 == sf2 == sf3 + ), f"Expected all StringFields to be equal, but they are not (sf1={sf1!r}, sf2={sf2!r}, sf3={sf3!r})" + + def test_normalize_with_different_lowercase_settings(self): + """Test normalize with different lowercase settings.""" + sf = StringField("TEST") + norm1 = sf.normalize(lowercase=True) + norm2 = sf.normalize(lowercase=False) + assert norm1 == "test", f"Expected norm1 to be 'test', but got {norm1!r}" + assert norm2 == "TEST", f"Expected norm2 to be 'TEST', but got {norm2!r}" + + def test_string_concatenation(self): + """Test that StringField can be concatenated like a string.""" + sf1 = StringField("hello") + sf2 = StringField("world") + result = sf1 + " " + sf2 + assert ( + result == "hello world" + ), f"Expected result to be 'hello world', but got {result!r}" + # UserString concatenation returns a new instance of the same class + assert isinstance( + result, StringField + ), f"Expected result to be a StringField instance, but got {type(result)}" + assert ( + result.data == "hello world" + ), f"Expected result.data to be 'hello world', but got {result.data!r}" diff --git a/tests/unit/test_unit.py b/tests/unit/test_unit.py new file mode 100644 index 0000000..8ff4dfa --- /dev/null +++ b/tests/unit/test_unit.py @@ -0,0 +1,271 @@ +import math + +import pytest + +from flowmapper.unit import UnitField + + +def test_equals_mass(): + u1 = UnitField("kg") + u2 = UnitField("kilogram") + + assert ( + u1 == u2 + ), f"Expected u1 to equal u2, but they are not equal (u1={u1!r}, u2={u2!r})" + + +def test_energy(): + u1 = UnitField("kilowatt hour") + u2 = UnitField("MJ") + assert u1.compatible( + u2 + ), f"Expected u1 to be compatible with u2, but they are not (u1={u1!r}, u2={u2!r})" + assert ( + u1.conversion_factor(u2) == 3.6 + ), f"Expected u1.conversion_factor(u2) to be 3.6, but got {u1.conversion_factor(u2)}" + + +def test_enrichment(): + u1 = UnitField("SWU") + u2 = UnitField("tonne * SW") + assert u1.compatible( + u2 + ), f"Expected u1 to be compatible with u2, but they are not (u1={u1!r}, u2={u2!r})" + assert ( + u1.conversion_factor(u2) == 1e-3 + ), f"Expected u1.conversion_factor(u2) to be 1e-3, but got {u1.conversion_factor(u2)}" + + +def test_natural_gas(): + u1 = UnitField("nm3") + u2 = UnitField("sm3") + assert u1.compatible( + u2 + ), f"Expected u1 to be compatible with u2, but they are not (u1={u1!r}, u2={u2!r})" + + +def test_livestock(): + u1 = UnitField("LU") + u2 = UnitField("livestock unit") + assert ( + u1.normalize() == u2.normalize() + ), f"Expected u1 to equal u2, but they are not equal (u1={u1!r}, u2={u2!r})" + + +def test_freight(): + u1 = UnitField("kilogram * km") + u2 = UnitField("tkm") + assert ( + u1.conversion_factor(u2) == 1e-3 + ), f"Expected u1.conversion_factor(u2) to be 1e-3, but got {u1.conversion_factor(u2)}" + + +def test_vehicular_travel(): + u1 = UnitField("vehicle * m") + u2 = UnitField("vkm") + assert ( + u1.conversion_factor(u2) == 1e-3 + ), f"Expected u1.conversion_factor(u2) to be 1e-3, but got {u1.conversion_factor(u2)}" + + +def test_person_travel(): + u1 = UnitField("person * m") + u2 = UnitField("pkm") + assert ( + u1.conversion_factor(u2) == 1e-3 + ), f"Expected u1.conversion_factor(u2) to be 1e-3, but got {u1.conversion_factor(u2)}" + + +def test_conversion_factor(): + u1 = UnitField("mg") + u2 = UnitField("kg") + actual = u1.conversion_factor(u2) + assert actual == 1e-06, f"Expected actual to be 1e-06, but got {actual}" + + +def test_nan_conversion_factor(): + u1 = UnitField("bq") + u2 = UnitField("kg") + actual = u1.conversion_factor(u2) + assert math.isnan(actual), f"Expected actual to be NaN, but got {actual}" + + +def test_complex_conversions(): + u1 = UnitField("square_meter_year / t") + u2 = UnitField("(meter ** 2 * month) / kg") + assert ( + u1.conversion_factor(u2) == 0.012 + ), f"Expected u1.conversion_factor(u2) to be 0.012, but got {u1.conversion_factor(u2)}" + + +class TestUnitFieldNormalize: + """Test UnitField normalize method.""" + + def test_normalize_with_valid_unit(self): + """Test normalize with valid unit.""" + u = UnitField("kg") + normalized = u.normalize() + assert ( + normalized == "kilogram" + ), f"Expected normalized to be 'kilogram', but got {normalized!r}" + assert isinstance( + normalized, UnitField + ), f"Expected normalized to be a UnitField instance, but got {type(normalized)}" + + def test_normalize_with_mapped_unit(self): + """Test normalize with unit that needs mapping.""" + # This tests the UNIT_MAPPING functionality + u = UnitField("kilogram") + normalized = u.normalize() + # The unit should be normalized through UNIT_MAPPING if applicable + assert isinstance( + normalized, UnitField + ), f"Expected normalized to be a UnitField instance, but got {type(normalized)}" + + def test_normalize_raises_error_undefined_unit(self): + """Test normalize raises error for undefined unit.""" + u = UnitField("unknown_unit_xyz") + with pytest.raises(ValueError, match="is unknown"): + u.normalize() + + +class TestUnitFieldEq: + """Test UnitField __eq__ method.""" + + def test_eq_with_same_data(self): + """Test equality with same data.""" + u1 = UnitField("kg") + u2 = UnitField("kg") + assert ( + u1 == u2 + ), f"Expected u1 to equal u2, but they are not equal (u1={u1!r}, u2={u2!r})" + + def test_eq_with_different_data_same_unit(self): + """Test equality with different data but same unit (conversion_factor == 1).""" + u1 = UnitField("kg") + u2 = UnitField("kilogram") + assert ( + u1 == u2 + ), f"Expected u1 to equal u2, but they are not equal (u1={u1!r}, u2={u2!r})" + + def test_eq_with_different_units(self): + """Test equality with different units.""" + u1 = UnitField("kg") + u2 = UnitField("g") + assert ( + u1 != u2 + ), f"Expected u1 to not equal u2, but they are equal (u1={u1!r}, u2={u2!r})" + + def test_eq_with_string(self): + """Test equality with string.""" + u = UnitField("kg") + assert u == "kg", f"Expected u to equal 'kg', but they are not equal (u={u!r})" + assert u != "g", f"Expected u to not equal 'g', but they are equal (u={u!r})" + + def test_eq_with_other_type(self): + """Test equality with other types.""" + u = UnitField("kg") + assert u != 123, f"Expected u to not equal 123, but they are equal (u={u!r})" + assert u != None, f"Expected u to not equal None, but they are equal (u={u!r})" + assert u != [], f"Expected u to not equal [], but they are equal (u={u!r})" + + +class TestUnitFieldCompatible: + """Test UnitField compatible method.""" + + def test_compatible_with_compatible_units(self): + """Test compatible with compatible units.""" + u1 = UnitField("kg") + u2 = UnitField("g") + assert u1.compatible( + u2 + ), f"Expected u1 to be compatible with u2, but they are not (u1={u1!r}, u2={u2!r})" + + def test_compatible_with_incompatible_units(self): + """Test compatible with incompatible units.""" + u1 = UnitField("kg") + u2 = UnitField("meter") + assert not u1.compatible( + u2 + ), f"Expected u1 to not be compatible with u2, but they are (u1={u1!r}, u2={u2!r})" + + def test_compatible_with_same_unit(self): + """Test compatible with same unit.""" + u1 = UnitField("kg") + u2 = UnitField("kg") + assert u1.compatible( + u2 + ), f"Expected u1 to be compatible with u2, but they are not (u1={u1!r}, u2={u2!r})" + + def test_compatible_with_non_unitfield(self): + """Test compatible with non-UnitField type.""" + u1 = UnitField("kg") + # Strings are now supported and work with compatible() + assert u1.compatible( + "kg" + ), f"Expected u1 to be compatible with 'kg' string (strings are now supported), but it is not (u1={u1!r})" + # Non-string, non-UnitField types should return False + assert not u1.compatible( + 123 + ), f"Expected u1 to not be compatible with 123, but it is (u1={u1!r})" + + +class TestUnitFieldConversionFactor: + """Test UnitField conversion_factor method.""" + + def test_conversion_factor_with_same_data(self): + """Test conversion_factor with same data.""" + u1 = UnitField("kg") + u2 = UnitField("kg") + result = u1.conversion_factor(u2) + assert result == 1.0, f"Expected conversion_factor to be 1.0, but got {result}" + + def test_conversion_factor_with_non_unitfield(self): + """Test conversion_factor with non-UnitField type.""" + u1 = UnitField("kg") + # Strings are now supported and work with conversion_factor() + result = u1.conversion_factor("kg") + assert ( + result == 1.0 + ), f"Expected conversion_factor to be 1.0 for same unit string, but got {result}" + # Non-string, non-UnitField types should return NaN + result2 = u1.conversion_factor(123) + assert math.isnan( + result2 + ), f"Expected conversion_factor to be NaN for non-UnitField, non-string type, but got {result2}" + + def test_conversion_factor_with_undefined_unit(self): + """Test conversion_factor with undefined unit.""" + u1 = UnitField("kg") + u2 = UnitField("unknown_unit_xyz") + result = u1.conversion_factor(u2) + assert math.isnan( + result + ), f"Expected conversion_factor to be NaN for undefined unit, but got {result}" + + def test_conversion_factor_with_dimensionality_error(self): + """Test conversion_factor with dimensionality error.""" + u1 = UnitField("kg") + u2 = UnitField("meter") + result = u1.conversion_factor(u2) + assert math.isnan( + result + ), f"Expected conversion_factor to be NaN for incompatible units, but got {result}" + + def test_conversion_factor_zero_to_one(self): + """Test conversion_factor from zero to one.""" + u1 = UnitField("mg") + u2 = UnitField("kg") + result = u1.conversion_factor(u2) + assert ( + result == 1e-06 + ), f"Expected conversion_factor to be 1e-06, but got {result}" + + def test_conversion_factor_one_to_zero(self): + """Test conversion_factor from one to zero.""" + u1 = UnitField("kg") + u2 = UnitField("mg") + result = u1.conversion_factor(u2) + assert ( + result == 1e06 + ), f"Expected conversion_factor to be 1e06, but got {result}" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..5e93143 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,250 @@ +"""Unit tests for FlowTransformationContext.""" + +from copy import copy + +import pytest + +from flowmapper.domain import Flow, NormalizedFlow +from flowmapper.utils import FlowTransformationContext + + +class TestFlowTransformationContext: + """Test FlowTransformationContext context manager.""" + + def test_single_function_applies_transformation(self): + """Test that a single function is applied on entry.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + flows = [nf] + + def transform_func(flows): + for flow in flows: + flow.update_current(name="Modified name") + return flows + + with FlowTransformationContext(flows, transform_func) as modified_flows: + assert ( + modified_flows[0].current.name.data == "Modified name" + ), "Expected flow to be modified in context" + assert ( + flows[0].current.name.data == "Modified name" + ), "Expected original flows list to be modified" + + # After exit, flows should be reset + assert ( + flows[0].current.name.data == normalized.name.data + ), "Expected flow to be reset after context exit" + + def test_enter_returns_modified_flows(self): + """Test that __enter__ returns the modified flows list.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + flows = [nf] + + def transform_func(flows): + for flow in flows: + flow.update_current(name="Modified") + return flows + + context = FlowTransformationContext(flows, transform_func) + returned_flows = context.__enter__() + + assert ( + returned_flows is flows + ), "Expected __enter__ to return the same flows list object" + assert ( + returned_flows[0].current.name.data == "Modified" + ), "Expected returned flows to be modified" + + context.__exit__(None, None, None) + + def test_reset_on_exit(self): + """Test that flows are reset to normalized state on exit.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + flows = [nf] + + def transform_func(flows): + for flow in flows: + flow.update_current(name="Modified", unit="g", context="water") + return flows + + with FlowTransformationContext(flows, transform_func): + # Verify modifications + assert flows[0].current.name.data == "Modified" + assert flows[0].current.unit.data == "g" + assert flows[0].current.context.value == "water" + + # After exit, all should be reset + assert ( + flows[0].current.name.data == normalized.name.data + ), "Expected name to be reset" + assert ( + flows[0].current.unit.data == normalized.unit.data + ), "Expected unit to be reset" + assert ( + flows[0].current.context.value == normalized.context.value + ), "Expected context to be reset" + + def test_reset_on_exception(self): + """Test that flows are reset even when an exception occurs.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + flows = [nf] + + def transform_func(flows): + for flow in flows: + flow.update_current(name="Modified") + return flows + + try: + with FlowTransformationContext(flows, transform_func): + assert flows[0].current.name.data == "Modified" + raise ValueError("Test exception") + except ValueError: + pass + + # After exception, flows should still be reset + assert ( + flows[0].current.name.data == normalized.name.data + ), "Expected flow to be reset even after exception" + + def test_function_returns_modified_list(self): + """Test that functions can return a modified list.""" + data1 = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + data2 = { + "name": "Water", + "context": "air", + "unit": "kg", + } + original1 = Flow.from_dict(data1) + original2 = Flow.from_dict(data2) + normalized1 = original1.normalize() + normalized2 = original2.normalize() + nf1 = NormalizedFlow( + original=original1, normalized=normalized1, current=copy(normalized1) + ) + nf2 = NormalizedFlow( + original=original2, normalized=normalized2, current=copy(normalized2) + ) + flows = [nf1, nf2] + + def filter_func(flows): + # Return only flows with "carbon" in name + filtered = [f for f in flows if "carbon" in f.current.name.data.lower()] + for flow in filtered: + flow.update_current(name="Filtered") + return filtered + + with FlowTransformationContext(flows, filter_func) as modified_flows: + assert ( + len(modified_flows) == 1 + ), "Expected filtered list to have one element" + assert ( + modified_flows[0].current.name.data == "Filtered" + ), "Expected filtered flow to be modified" + + # Original flows list should still have both flows + assert len(flows) == 2, "Expected original flows list to be unchanged" + + def test_multiple_flows_all_reset(self): + """Test that all flows in the list are reset.""" + data1 = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + data2 = { + "name": "Water", + "context": "air", + "unit": "kg", + } + original1 = Flow.from_dict(data1) + original2 = Flow.from_dict(data2) + normalized1 = original1.normalize() + normalized2 = original2.normalize() + nf1 = NormalizedFlow( + original=original1, normalized=normalized1, current=copy(normalized1) + ) + nf2 = NormalizedFlow( + original=original2, normalized=normalized2, current=copy(normalized2) + ) + flows = [nf1, nf2] + + def transform_func(flows): + for i, flow in enumerate(flows): + flow.update_current(name=f"Modified {i}") + return flows + + with FlowTransformationContext(flows, transform_func): + assert flows[0].current.name.data == "Modified 0" + assert flows[1].current.name.data == "Modified 1" + + # Both should be reset + assert ( + flows[0].current.name.data == normalized1.name.data + ), "Expected first flow to be reset" + assert ( + flows[1].current.name.data == normalized2.name.data + ), "Expected second flow to be reset" + + def test_no_functions(self): + """Test that context manager works with no functions.""" + data = { + "name": "Carbon dioxide", + "context": "air", + "unit": "kg", + } + original = Flow.from_dict(data) + normalized = original.normalize() + nf = NormalizedFlow( + original=original, normalized=normalized, current=copy(normalized) + ) + flows = [nf] + + with FlowTransformationContext(flows) as returned_flows: + assert returned_flows is flows, "Expected same flows list to be returned" + assert ( + returned_flows[0].current.name.data == normalized.name.data + ), "Expected flows to be unchanged" + + # Should still reset (though nothing changed) + assert ( + flows[0].current.name.data == normalized.name.data + ), "Expected flow to remain normalized"