diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index e7fd5d9..e55d694 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -10,6 +10,10 @@ on: jobs: build-upload-docs: runs-on: ubuntu-latest + # Only one deployment at a time, otherwise the pushes to data.modm.io race. + concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -22,6 +26,7 @@ jobs: - name: Install modm-data with docs dependencies run: | pip3 install -r tools/requirements.txt -e ".[docs]" + pip3 install git+https://github.com/modm-ext/modm-pinout.git@main - name: Install ARM toolchain and sources for the SVD files run: | @@ -50,6 +55,10 @@ jobs: git config --global user.email "bot@modm.io" git config --global user.name "modm update bot" cd docs/data.modm.io + # The generated content replaces the repository wholesale, so rebase + # onto whatever is on the remote instead of merging into it. + git fetch origin main + git reset --soft origin/main git add -A git diff-index --quiet HEAD || git commit -m "Update" git push origin main diff --git a/.github/workflows/test-svd.yml b/.github/workflows/test-svd.yml index b12cf6a..f8ad001 100644 --- a/.github/workflows/test-svd.yml +++ b/.github/workflows/test-svd.yml @@ -54,10 +54,33 @@ jobs: path: svd-archive/ retention-days: 7 - - name: Zip SVD Files + - name: Analyze Peripheral Variants + run: | + curl -fsSL --create-dirs -o ext/cache/svd2variants/feature-names-gemma.json \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/feature-names-gemma.json || true + python3 -m modm_data.svd2variants --all --input svd-archive/ --names ext/cache/svd2variants/feature-names-gemma.json \ + --no-generate --output variants-archive/ + + - name: Upload Variant Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: variant-logs + path: log/stmicro/variants + retention-days: 1 + + - name: Upload Peripheral Variants + uses: actions/upload-artifact@v4 + with: + name: variants-archive + path: variants-archive/ + retention-days: 7 + + - name: Zip SVD Files and Peripheral Variants if: ${{ github.event_name == 'release' && github.event.action == 'published' }} run: | zip -r svd-archive.zip svd-archive/ + zip -r variants-archive.zip variants-archive/ - name: Upload to Github Release uses: softprops/action-gh-release@v2 @@ -65,3 +88,4 @@ jobs: with: files: | svd-archive.zip + variants-archive.zip diff --git a/CHANGELOG.md b/CHANGELOG.md index 72cc27a..a7acc32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- Merge the register maps of all STM32 peripheral instances into the smallest + set of binary, source and similar compatible variants and publish them with + the Peripheral Variant Explorer. +- Add the register access to the SVD files reconstructed from the CMSIS headers. + ## 0.0.4 - Move the data extraction pipelines of all vendors from modm-devices. diff --git a/README.md b/README.md index ff279e4..4b261af 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,9 @@ flowchart LR B(HTML) -->|html| C C(Python) -->|owl| E D(CMSIS-SVD) -->|cmsis-svd| C + D -->|svd2variants| K E[OWL] + K[Peripheral Variants] F(CMSIS Header) -->|header2svd| D I(Open Pin Data) -->|cubemx| C G(CubeMX) -->|cubemx| C diff --git a/docs/.gitignore b/docs/.gitignore index f2e9bc7..c81190d 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -2,6 +2,7 @@ src/api/ src/pinout/ src/svd/ +src/variants/ src/index.md src/source/overview.md src/pipeline/overview.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ff1e072..68a7c45 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -87,6 +87,8 @@ nav: - HTML to SVD: pipeline/html2svd.md #- HTML to OWL: pipeline/html2owl.md - Header to SVD: pipeline/header2svd.md + - SVD to Variants: pipeline/svd2variants.md - API Reference: api/modm_data.html - Pinouts: pinout/index.html - SVD Explorer: svd/index.html + - Peripheral Variants: variants/index.html diff --git a/docs/src/pipeline/svd2variants.md b/docs/src/pipeline/svd2variants.md new file mode 100644 index 0000000..9d9506e --- /dev/null +++ b/docs/src/pipeline/svd2variants.md @@ -0,0 +1,163 @@ +# SVD to Peripheral Variants Pipeline + +Every STM32 device has its own register map, but the peripherals inside them +are reused across the entire device range. This pipeline answers how many +*different* implementations of a peripheral actually exist by merging the +register maps of all peripheral instances of all devices into as few variants +as possible, see `modm_data.svd2variants`. + +The peripheral instances are grouped by their CMSIS structure type, which the +[header to SVD pipeline](header2svd.md) writes into the `groupName` element, +so that `TIM1` and `TIM17` are compared with each other, but not with `SPI1`. + +Some registers belong to the device rather than to the peripheral and are left +out of the comparison, see `modm_data.svd2variants.model.IGNORED_REGISTERS`. +The timer option registers `OR`, `OR1`-`OR3`, `AF1`, `AF2` and `TISEL` route +device signals to the timer inputs, e.g. which comparator drives the break +input, so their layout follows the device family and would otherwise split the +timers into one variant per family. + +## What Makes Two Register Maps Different + +Neither the names nor the addresses alone can decide whether two register maps +describe the same silicon, see `modm_data.svd2variants.merge`: + +- ST renames registers and bit fields without changing the layout, most + visibly between the STM32H7x and STM32H7[RS]x devices. Comparing names alone + reports these as different implementations, which they are not. +- ST reuses layouts for entirely different registers. The STM32F1 `GPIO_CRL` + and the `GPIO_MODER` of every other device both start with two-bit fields at + the same positions, so comparing addresses alone merges the two GPIO + implementations into one, which they are not. + +Therefore every element is compared by both keys, but not symmetrically: + +- A **name at two different locations** is a *conflict*: no single driver can + address both maps, so they are different variants. This is what separates + the STM32F1 GPIO (`IDR` at `0x08`) from every other GPIO (`IDR` at `0x10`). +- A **location with two different names** is a *rename*: the silicon is the + same and every name is kept as an alias. +- A bit field that only grew into the reserved bits above it is an optional + extension instead of a conflict, since ST widens fields like the `IWDG_PR` + prescaler in place. Pass `--strict` to treat this as a conflict instead. + +Unrelated elements never share a name, so two more rules catch what the names +cannot: + +- Two **differently laid out bit fields that share bits** are a conflict, + unless they are alternates of each other within one register map. Without + this, the STM32L1 `COMP_CSR_OUTSEL[23:21]` and the STM32G0 + `COMP_CSR_BLANKING[24:20]` merge into one register, each an optional feature + of the other. +- A **rename only counts inside the same register**, meaning the registers + share a name or at least one identically named bit field. The STM32F373 + `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` share a bit, but + nothing else, so the bit was reused for something else. This mostly matters + for binary compatibility, since in source mode the register name is already + part of every location. + +Everything else is a *population* difference: an element that only exists in +some of the instances becomes an optional feature of the merged map. A feature +is not a single register or bit field, but all elements that appear in exactly +the same instances, since those are enabled and disabled together, for example +the clock enable bits in `RCC` of a GPIO port that only the larger packages +bond out. + +## Binary, Source and Similar Compatibility + +Many register maps differ only in *where* a register sits, while the register +itself is unchanged. Which of the two matters depends on how the hardware is +addressed, so the analysis runs in three modes: + +- `binary`: elements are located by the **address** of their register, so + moving a register to another offset is a conflict. This is what a driver + that pokes at fixed addresses needs. +- `source`: elements are located by the **name** of their register, so a + register may move as long as it keeps its name and its layout. This is what + code written against a CMSIS header needs, since the structure member + abstracts the offset away. +- `similar`: like `source`, but differences that only exist in the + documentation are tolerated, see `modm_data.svd2variants.merge.documentation_only`. + This is a heuristic for how many implementations are *essentially* the same. + +Neither mode is strictly coarser than the other. Source compatibility merges +the two GPIO implementations into one, because the STM32F1 `IDR` at `0x08` and +the `IDR` at `0x10` of every other device have the same bit fields. But it +also splits `COMP` further, because several devices have a `CSR` register with +the same name and a different layout, which only collides once the name is +what identifies the register. + +The `similar` mode tolerates a register declared with another width (the +STM32F0 declares `USART_RDR` as `uint16_t`), a bit field with the same name +over overlapping bits, and a bit field described as smaller bit fields. The last +one is only tolerated if the names are related, like `COMP_CSR_BLANKING` and +`COMP_CSR_COMPxBLANKING`, or if the smaller bit fields tile one edge of a bit +field of at least four bits, either as a value split into at least two parts +like `USART_BRR_BRR` into `DIV_MANTISSA` and `DIV_FRACTION`, or as flags that +take up at most a quarter of it like `TIM_CNT_UIFCPY` in the top bit of +`TIM_CNT_CNT`. Unrelated bit fields nest by coincidence surprisingly often, e.g. +the STM32F4 `FLASH_OPTCR1_nWRP[27:16]` inside the STM32F7 +`FLASH_OPTCR1_BOOT_ADD1[31:16]`, so anything else is still a conflict. + +```bash +# Only compare by register address +python3 -m modm_data.svd2variants --all --mode binary +``` + +## Naming the Features + +A feature is a set of elements that appear in exactly the same instances, which +is a mechanical definition and says nothing about what the feature *is*. Most +of them do share something lexically, since ST names and documents related +elements alike, so a name is derived from the shared name prefix, the shared +description phrase or the registers involved, see `modm_data.svd2variants.naming`. + +That misses the semantics though: the seventeen bit fields of the first +capture/compare channel of a timer share no name and no description, but they +are obviously "Channel 1". A small language model names those much better, so +the names are generated into a lookup table keyed by the *content* of the +feature, see `modm_data.svd2variants.annotate`. The table is stable across +runs, reviewable as a diff, and only features that actually changed need to be +asked again. Every feature is named by the first of these that works: + +1. the lookup table of `--names`, which the homepage and the CI download from + the gist, so that they never need a model, +2. a local model, by default Gemma 4 26B-A4B served by LM Studio at + `http://localhost:1234/v1`, which only new features are sent to, +3. the heuristic. + +The prompt matters more than the size of the model. Keeping the numbers of +similar elements as a range turns "High line interrupt masking" into "Lines 28 +to 30 interrupt masking", and asking for a different name per feature stops +four ADC features from all becoming "Multimode and dual mode support". +Conversely, naming the vague words to avoid produced *more* of them. + +```bash +# Name the features that are missing from the table with a local model and update it +python3 -m modm_data.svd2variants --all --names feature-names.json +# Name them with the claude command line tool instead +python3 -m modm_data.svd2variants --all --names feature-names.json --claude +# Use another local model or server +python3 -m modm_data.svd2variants --all --names feature-names.json \ + --endpoint http://localhost:1234/v1 --model qwen/qwen3-4b-2507 +# Only use the table, as the CI does +python3 -m modm_data.svd2variants --all --names feature-names.json --no-generate +``` + +The homepage and the CI workflow currently use the heuristic names only. + +## Selective Analysis + +The reports are written to `log/stmicro/variants//*.txt`. + +```bash +# Analyze only these peripheral groups +python3 -m modm_data.svd2variants --group "GPIO" --group "TIM" +# Analyze all peripheral groups and write the JSON files +python3 -m modm_data.svd2variants --all --output docs/src/variants/ +``` + +The results are also published on the homepage with the +[Peripheral Variant Explorer](https://gist.github.com/salkinium/12a18032caa303697c6583937f6fcd16), +which loads the summary of both modes from `variants.json` and one file per +peripheral group from `binary/` and `source/`. diff --git a/pyproject.toml b/pyproject.toml index 84d97e1..e0e0fe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,11 +44,12 @@ dynamic = ["version"] "Changelog" = "https://github.com/modm-io/modm-data/blob/main/CHANGELOG.md" [project.optional-dependencies] +# Note: modm-pinout is not declared here, since PyPI rejects packages with +# direct URL dependencies. The docs workflow installs it separately. docs = [ "pdoc>=14,<15", "mkdocs>=1.5,<2", "mkdocs-material>=9.5,<10", - "modm-pinout @ git+https://github.com/modm-ext/modm-pinout.git@main", ] all = ["modm_data[docs]"] diff --git a/src/modm_data/header2svd/extract.py b/src/modm_data/header2svd/extract.py index 829c59b..c60708a 100644 --- a/src/modm_data/header2svd/extract.py +++ b/src/modm_data/header2svd/extract.py @@ -37,7 +37,7 @@ _CMSIS_PATH = ext_path("arm/cmsis/CMSIS/Core/Include") _CACHE_PATH = cache_path("cmsis/header2svd") -_VERSION = 2 +_VERSION = 3 @dataclass @@ -48,6 +48,10 @@ class Member: """Type name of the member, or `pointer` for pointer members.""" length: str | None """The array length expression or `None` if the member is not an array.""" + access: str = "read-write" + """`read-only` for members declared with `__I` or `__IM`, which expand to + `volatile const`. The `__O` qualifier expands to the same `volatile` as `__IO`, + so write-only members cannot be told apart, but ST does not use it.""" @dataclass @@ -152,6 +156,7 @@ def members(cls, name): if isinstance(mtype, ctypes.Array): length = "".join(token.value for token in mtype.size.tokens) mtype = mtype.array_of + access = "read-only" if getattr(mtype, "const", False) else "read-write" if isinstance(mtype, (ctypes.Pointer, ctypes.FunctionType)) or not hasattr(mtype, "typename"): if member.name: result.append(Member(member.name, "pointer", length)) @@ -171,7 +176,7 @@ def members(cls, name): result.append(Member(member.name, subtype, length)) continue if member.name: - result.append(Member(member.name, segment.name, length)) + result.append(Member(member.name, segment.name, length, access)) # Anonymous unions and structures without a member name for cid, inner_cls in inner.items(): if cid not in named: diff --git a/src/modm_data/header2svd/stmicro/memory_map.py b/src/modm_data/header2svd/stmicro/memory_map.py index 58e9621..0d3505f 100644 --- a/src/modm_data/header2svd/stmicro/memory_map.py +++ b/src/modm_data/header2svd/stmicro/memory_map.py @@ -509,12 +509,14 @@ def _bit_fields(register: _Register, fields: dict[str, str], bitfields: _BitFiel for position, width, index in bitfields.fields[macro]: split = len(bitfields.fields[macro]) > 1 fname = f"{name}_{index}" if index else name - candidates.append((fname, position, width, macro, split)) + candidates.append((fname, position, width, macro, split, name)) def bits(position, width): return set(range(position, position + width)) - # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0 + # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0. The parent + # may be one part of a non-contiguous bit field, e.g. CRYP_CR_ALGOMODE_AES_KEY in bits 5:3 of CRYP_CR_ALGOMODE, + # which also uses bit 19, otherwise the values replace the bit field itself. result = [] for candidate in candidates: cbits = bits(*candidate[1:3]) @@ -522,7 +524,10 @@ def bits(position, width): ( c for c in candidates - if c is not candidate and candidate[0].startswith(c[0] + "_") and cbits <= bits(*c[1:3]) and not c[4] + if c is not candidate + and candidate[0] != c[0] + and candidate[0].startswith(c[5] + "_") + and cbits <= bits(*c[1:3]) ), None, ) @@ -547,7 +552,7 @@ def bits(position, width): # Prefer non-alias and non-split bit fields in the order of definition result.sort(key=lambda c: (c[3] in bitfields.alias, c[4], bitfields.order.get(c[3], 0), c[1])) used, alternate = {}, [] - for fname, position, width, macro, split in result: + for fname, position, width, macro, split, _name in result: overlap = next((used[b] for b in bits(position, width) if b in used), None) if overlap is not None and fname not in register.fields: # A narrow field at the edge of a much wider field, e.g. TIM_CNT_UIFCPY in TIM_CNT_CNT @@ -769,6 +774,7 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> interrupts = _interrupts(data, {re.sub(r"_NS$", "", pname) for pname in peripherals}, report) device = Device(name or data.header.stem, compatible=data.defines[:1]) + access = {(typedef, member.name): member.access for typedef, members in data.structs.items() for member in members} signatures, addresses = {}, {} for pname, (address, ptype, registers) in sorted(peripherals.items(), key=lambda p: (p[1][0], p[0])): # Registers of sub-instances with colliding names are prefixed with the sub-instance name @@ -791,6 +797,8 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> # The non-secure instances use the plain name peripheral = Peripheral(re.sub(r"_NS$", "", pname), ptype, address, parent=device) peripheral.description = data.type_descriptions.get(ptype, "") + if ptype: + peripheral.group = re.sub(r"_?TypeDef$", "", ptype) peripheral.interrupts = interrupts.get(peripheral.name, []) if (alternate := addresses.setdefault(address, peripheral.name)) != peripheral.name: peripheral.alternate = alternate @@ -800,6 +808,7 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> offsets = {} for register in registers: treg = Register(register.name, register.offset, register.size, parent=peripheral) + treg.access = access.get((register.type, register.member), "read-write") treg.description = data.member_descriptions.get((register.type, register.member), "") if register.dim: treg.dim = register.dim diff --git a/src/modm_data/svd/read.py b/src/modm_data/svd/read.py index 3cdd781..f3c544d 100644 --- a/src/modm_data/svd/read.py +++ b/src/modm_data/svd/read.py @@ -3,7 +3,7 @@ import re from lxml import etree -from .model import Device, Peripheral, Register, BitField +from .model import Device, Peripheral, Register, BitField, EnumeratedValue def _int(node, tag, default=None): @@ -12,6 +12,12 @@ def _int(node, tag, default=None): return int(text.strip().replace("#", "0b"), 0) +def _text(node, tag) -> str | None: + if (text := node.findtext(tag)) is None: + return None + return " ".join(text.split()) + + def _bit_range(node) -> tuple[int, int]: if (offset := _int(node, "bitOffset")) is not None: return offset, _int(node, "bitWidth", 1) @@ -21,6 +27,17 @@ def _bit_range(node) -> tuple[int, int]: return lsb, msb - lsb + 1 +def _add_bit_fields(rnode, register): + for fnode in rnode.iter("field"): + field = BitField(fnode.findtext("name"), *_bit_range(fnode), parent=register) + field.description = _text(fnode, "description") + for vnode in fnode.iter("enumeratedValue"): + if (value := _text(vnode, "value")) is None: + continue + enum = EnumeratedValue(vnode.findtext("name"), value.replace("#", "0b"), parent=field) + enum.description = _text(vnode, "description") + + def read_svd(path) -> Device: """ Reads the peripherals, registers and bit fields of a CMSIS-SVD file. @@ -37,10 +54,19 @@ def read_svd(path) -> Device: for node in root.iter("peripheral"): name = node.findtext("name") registers[name] = node.find("registers") - peripherals.append((name, node.get("derivedFrom"), _int(node, "baseAddress"))) + peripherals.append(node) - for name, derived_from, address in peripherals: - peripheral = Peripheral(name, derived_from, address, parent=device) + for node in peripherals: + name = node.findtext("name") + derived_from = node.get("derivedFrom") + peripheral = Peripheral(name, _text(node, "groupName"), _int(node, "baseAddress"), parent=device) + peripheral.group = _text(node, "groupName") + peripheral.description = _text(node, "description") + peripheral.alternate = _text(node, "alternatePeripheral") + peripheral.interrupts = [ + (inode.findtext("name"), _int(inode, "value"), _text(inode, "description")) + for inode in node.iter("interrupt") + ] if derived_from: peripheral.derived_from = derived_from if (nodes := registers[name]) is None and derived_from: @@ -49,8 +75,14 @@ def read_svd(path) -> Device: continue for rnode in nodes.iter("register"): size = _int(rnode, "size", 32) // 8 - register = Register(rnode.findtext("name"), _int(rnode, "addressOffset"), size, parent=peripheral) - for fnode in rnode.iter("field"): - BitField(fnode.findtext("name"), *_bit_range(fnode), parent=register) + rname = rnode.findtext("name") + register = Register(rname.replace("[%s]", ""), _int(rnode, "addressOffset"), size, parent=peripheral) + register.description = _text(rnode, "description") + register.alternate = _text(rnode, "alternateRegister") + register.access = _text(rnode, "access") + if (dim := _int(rnode, "dim")) is not None: + register.dim = dim + register.dim_increment = _int(rnode, "dimIncrement", size) + _add_bit_fields(rnode, register) return device diff --git a/src/modm_data/svd/write.py b/src/modm_data/svd/write.py index 9fd0b00..a46e8e3 100644 --- a/src/modm_data/svd/write.py +++ b/src/modm_data/svd/write.py @@ -33,6 +33,8 @@ def _format_peripheral(xmlnode, treenode): _add_element(peripheral, "name", treenode.name) if description := getattr(treenode, "description", None): _add_element(peripheral, "description", description) + if group := getattr(treenode, "group", None): + _add_element(peripheral, "groupName", group) if alternate := getattr(treenode, "alternate", None): _add_element(peripheral, "alternatePeripheral", alternate) _add_element(peripheral, "baseAddress", hex(treenode.address)) @@ -62,6 +64,8 @@ def _format_register(xmlnode, treenode): _add_element(register, "alternateRegister", alternate) _add_element(register, "addressOffset", hex(treenode.offset)) _add_element(register, "size", hex(treenode.width * 8)) + if access := getattr(treenode, "access", None): + _add_element(register, "access", access) if treenode.children: return _add_element(register, "fields") else: diff --git a/src/modm_data/svd2variants/__init__.py b/src/modm_data/svd2variants/__init__.py new file mode 100644 index 0000000..6594498 --- /dev/null +++ b/src/modm_data/svd2variants/__init__.py @@ -0,0 +1,46 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Peripheral Variants from SVD Files + +Every STM32 device has its own register map, but the peripherals inside them +are reused across the entire device range. This module answers how many +*different* implementations of a peripheral actually exist by merging the +register maps of all peripheral instances of all devices into as few variants +as possible. + +Each variant is one silicon implementation, described by a common core that +every instance implements and a set of optional features that only some +instances implement. Two variants of the same peripheral cannot be merged, +since they contradict each other, see `modm_data.svd2variants.merge` for what +exactly makes two register maps contradict. +""" + +from .model import Field, Register, Shape, Instance, group_name +from .merge import MODES, ElementMap, Conflict, Rename, Difference, compare, scope_of +from .analyze import Feature, Member, Variant, Group, variants_of, conflicts_between +from .read import instances_of_svd, instances_of_svds + +__all__ = [ + "Field", + "Register", + "Shape", + "Instance", + "group_name", + "MODES", + "ElementMap", + "scope_of", + "Conflict", + "Rename", + "Difference", + "compare", + "Feature", + "Member", + "Variant", + "Group", + "variants_of", + "conflicts_between", + "instances_of_svd", + "instances_of_svds", +] diff --git a/src/modm_data/svd2variants/__main__.py b/src/modm_data/svd2variants/__main__.py new file mode 100644 index 0000000..31530b4 --- /dev/null +++ b/src/modm_data/svd2variants/__main__.py @@ -0,0 +1,203 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +import re +import logging +import argparse +from pathlib import Path +from collections import defaultdict + +from modm_data.svd2variants import instances_of_svds, variants_of, conflicts_between +from modm_data.svd2variants.merge import MODES +from modm_data.svd2variants.write import write_variants +from modm_data.svd2variants.annotate import ( + LOCAL_MODEL, + MODEL, + feature_key, + generate_names, + local_endpoint, + read_names, + write_names, +) +from modm_data.utils import ext_path + +LOGGER = logging.getLogger("svd2variants") + + +def _report(group) -> str: + lines = [ + f"{group.name}: {len(group.variants)} {group.mode} compatible variants of {group.shapes} register maps", + "", + ] + for index, variant in enumerate(group.variants): + registers = {location for location in variant.elements.names if location[0] == "R"} + lines.append( + f"[{index}] {len(variant.instances)} instances on {len(variant.devices)} devices, " + f"{len(variant.members)} register maps, {len(registers)} registers, " + f"{len(variant.core)} core elements, {len(variant.features)} features" + ) + lines.append(f" instances: {' '.join(sorted(variant.names))}") + lines.append(f" devices: {' '.join(sorted(variant.devices))}") + for number, feature in enumerate(variant.features): + lines.append( + f" feature {number} ({feature.name}): {len(feature.locations)} elements in " + f"{len(feature.shapes)}/{len(variant.members)} register maps, max {feature.instances} instances" + ) + if variant.renames: + lines.append(f" renamed: {len(set(map(str, variant.renames)))}") + lines += [f" {rename}" for rename in sorted({str(r) for r in variant.renames})] + lines.append("") + for (left, right), conflicts in conflicts_between(group.variants, group.mode).items(): + lines.append(f"Conflicts between [{left}] and [{right}]: {len(conflicts)}") + lines += [f" {conflict}" for conflict in conflicts] + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser( + description="Merges the register maps of all peripheral instances into as few variants as possible." + ) + parser.add_argument( + "--group", + type=str, + default=[], + action="append", + help="Regex pattern of peripheral groups to analyze, e.g. GPIO.", + ) + parser.add_argument("--all", action="store_true", default=False, help="Analyze all peripheral groups.") + parser.add_argument( + "--input", + type=Path, + default=None, + help="Folder with the SVD files, defaults to ext/stmicro/svd.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Folder for the variants.json and the per group JSON files.", + ) + parser.add_argument( + "--strict", + action="store_true", + default=False, + help="Treat a bit field that only grew into the reserved bits above it as a conflict.", + ) + parser.add_argument( + "--mode", + choices=[*MODES, "all"], + default="all", + help="Locate elements by register address (binary), by register name (source), or by register name while tolerating differences in the documentation (similar).", + ) + parser.add_argument( + "--names", + type=Path, + default=None, + help="JSON lookup table of generated feature names, which takes precedence over the heuristic.", + ) + parser.add_argument( + "--no-generate", + action="store_true", + default=False, + help="Only use the names of --names, even if a model is available to name the rest.", + ) + parser.add_argument( + "--claude", + action="store_true", + default=False, + help="Name the missing features with the claude command line tool instead of a local server.", + ) + parser.add_argument("--model", type=str, default=None, help="The model that names the features.") + parser.add_argument( + "--endpoint", + type=str, + default=None, + help=f"URL of an OpenAI compatible server, defaults to LM Studio serving {LOCAL_MODEL}.", + ) + parser.add_argument( + "--workers", + type=int, + default=None, + help="Number of concurrent requests, defaults to 6 for claude and 1 for a local server.", + ) + parser.add_argument("-v", "--verbose", action="count", default=0) + args = parser.parse_args() + logging.basicConfig(level=[logging.WARNING, logging.INFO, logging.DEBUG][min(args.verbose, 2)]) + + paths = sorted((args.input or ext_path("stmicro/svd")).glob("*.svd")) + if not paths: + print("No SVD files found!") + return False + instances = instances_of_svds(paths) + names = [n for n in instances if args.all or any(re.match(p, n) for p in args.group)] + if not names: + print("No matching peripheral groups found!") + return False + + table = read_names(args.names) + modes = { + mode: [variants_of(name, instances[name], widening=not args.strict, mode=mode, names=table) for name in names] + for mode in (MODES if args.mode == "all" else [args.mode]) + } + + if not args.no_generate: + jobs = defaultdict(list) + seen = set() + for groups in modes.values(): + for group in groups: + for variant in group.variants: + for feature in variant.features: + key = feature_key(feature.elements) + if key not in table and key not in seen: + seen.add(key) + jobs[group.name].append((key, feature.elements)) + # The table of a previous run first, then a local model, then the heuristic + endpoint = None if args.claude else local_endpoint(args.endpoint, args.model or LOCAL_MODEL) + model = args.model or (MODEL if args.claude else LOCAL_MODEL) + if seen and (endpoint or args.claude): + print(f"Naming {len(seen)} features of {len(jobs)} peripheral groups with {model}...") + + def progress(result, done, total): + table.update(result) + if args.names: + write_names(table, args.names) + print(f" batch {done}/{total}: {len(result)} names", flush=True) + + workers = args.workers or (6 if args.claude else 1) + generated = generate_names(jobs, model, workers, progress, endpoint) + if args.names: + print(f"{len(table)} names in {args.names}") + # Name the features again, now that the table is filled + if generated: + modes = { + mode: [ + variants_of(name, instances[name], widening=not args.strict, mode=mode, names=table) + for name in names + ] + for mode in (MODES if args.mode == "all" else [args.mode]) + } + elif seen: + print(f"{len(seen)} features are named by the heuristic, no model available to name them.") + for mode, groups in modes.items(): + log_path = Path("log/stmicro/variants") / mode + log_path.mkdir(exist_ok=True, parents=True) + for group in groups: + (log_path / f"{group.name}.txt").write_text(_report(group)) + if args.output: + write_variants(modes, args.output) + + header = "".join(f"{mode + ' variants':>18}" for mode in modes) + print(f"{'group':24} {'instances':>9} {'maps':>6}{header}") + for group in sorted(next(iter(modes.values())), key=lambda g: (-len(g.variants), -g.shapes)): + counts = "".join( + f"{len(next(g for g in groups if g.name == group.name).variants):18}" for groups in modes.values() + ) + print(f"{group.name:24} {len(group.instances):9} {group.shapes:6}{counts}") + totals = "".join(f"{sum(len(g.variants) for g in groups):18}" for groups in modes.values()) + first = next(iter(modes.values())) + print(f"{'Total':24} {sum(len(g.instances) for g in first):9} {sum(g.shapes for g in first):6}{totals}") + return True + + +if __name__ == "__main__": + exit(0 if main() else 1) diff --git a/src/modm_data/svd2variants/analyze.py b/src/modm_data/svd2variants/analyze.py new file mode 100644 index 0000000..8ae047a --- /dev/null +++ b/src/modm_data/svd2variants/analyze.py @@ -0,0 +1,238 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Condensing Register Maps into Variants + +All instances of a peripheral group are first deduplicated into distinct +*shapes*, then greedily merged into *variants*: a variant is a set of shapes +that can be merged without a conflict, and therefore describes one silicon +implementation of the peripheral. + +The merged register map of a variant is split into a **core**, which every +instance of the variant implements, and a set of **features**, which are the +elements that only some instances implement. A feature is not a single +register or bit field, but every element that appears in exactly the same +instances, since those are enabled and disabled together, for example the +second half of the CAN filter banks or the clock enable bits of a peripheral +that is only bonded out on the larger packages. +""" + +import re +from dataclasses import dataclass, field as _field +from collections import defaultdict +from .model import Shape, Instance, LocationKey, IGNORED_REGISTERS +from .merge import ElementMap, Conflict, Rename, compare, register_location, field_location +from .naming import name_feature + + +@dataclass +class Feature: + """The elements of a variant that are implemented by the same instances.""" + + locations: set[LocationKey] + shapes: frozenset[int] = _field(repr=False) + instances: int = 0 + name: str = "" + elements: list = _field(default_factory=list, repr=False) + + @property + def registers(self) -> set[int]: + return {location[1] for location in self.locations if location[0] == "R"} + + +@dataclass +class Member: + """One distinct register map of a variant and every instance that has it.""" + + shape: Shape = _field(repr=False) + elements: ElementMap = _field(repr=False) + instances: list[Instance] = _field(default_factory=list, repr=False) + + @property + def devices(self) -> set[str]: + return {instance.device for instance in self.instances} + + +@dataclass +class Variant: + """One silicon implementation of a peripheral group.""" + + group: str + members: list["Member"] = _field(default_factory=list, repr=False) + elements: ElementMap = _field(default_factory=ElementMap, repr=False) + renames: list[Rename] = _field(default_factory=list, repr=False) + core: set[LocationKey] = _field(default_factory=set, repr=False) + features: list[Feature] = _field(default_factory=list, repr=False) + + @property + def shapes(self) -> list[Shape]: + return [member.shape for member in self.members] + + @property + def instances(self) -> list[Instance]: + return [instance for member in self.members for instance in member.instances] + + @property + def devices(self) -> set[str]: + return {instance.device for member in self.members for instance in member.instances} + + @property + def names(self) -> set[str]: + return {instance.name for member in self.members for instance in member.instances} + + def add(self, member: "Member", renames: list[Rename]): + self.members.append(member) + self.elements.update(member.elements) + self.renames += renames + + def described(self, mode: str) -> dict[LocationKey, tuple[str, str, str, str]]: + """:return: the kind, register name, name and description of every element.""" + described = {} + for member in self.members: + for register in member.shape.registers: + described.setdefault( + register_location(register, mode), ("R", register.name, register.name, register.description) + ) + for field in register.fields: + described.setdefault( + field_location(register, field, mode), ("F", register.name, field.name, field.description) + ) + return described + + def condense(self, mode: str = "binary", names: dict[str, str] = None): + """Splits the merged register map into the common core and the optional features.""" + presence = defaultdict(set) + instances = defaultdict(int) + for index, member in enumerate(self.members): + for location in member.elements.names: + presence[location].add(index) + instances[location] += len(member.instances) + total = len(self.members) + self.core = {location for location, shapes in presence.items() if len(shapes) == total} + features = defaultdict(set) + for location, shapes in presence.items(): + if len(shapes) != total: + features[frozenset(shapes)].add(location) + self.features = sorted( + ( + Feature(locations, shapes, max(instances[location] for location in locations)) + for shapes, locations in features.items() + ), + key=lambda f: (-f.instances, -len(f.locations)), + ) + described = self.described(mode) + for feature in self.features: + feature.elements = [described[ll] for ll in sorted(feature.locations) if ll in described] + feature.name = name_feature(feature.elements, names) + + +@dataclass +class Group: + """All instances of one peripheral type across all devices.""" + + name: str + instances: list[Instance] = _field(default_factory=list, repr=False) + variants: list[Variant] = _field(default_factory=list, repr=False) + mode: str = "binary" + ignored: list[str] = _field(default_factory=list) + """The names of the registers that were left out of the comparison.""" + + @property + def shapes(self) -> int: + return sum(len(variant.shapes) for variant in self.variants) + + @property + def devices(self) -> set[str]: + return {instance.device for instance in self.instances} + + +def _shapes(instances: list[Instance], ignored: re.Pattern = None) -> dict[Shape, list[Instance]]: + """Deduplicates the instances by their register map without the ignored registers.""" + shapes = defaultdict(list) + for instance in instances: + shapes[instance.shape.without(ignored) if ignored else instance.shape].append(instance) + # The largest register maps discriminate best and therefore seed the variants + return dict(sorted(shapes.items(), key=lambda s: (-len(s[0]), -len(s[1])))) + + +def variants_of( + name: str, instances: list[Instance], widening: bool = True, mode: str = "binary", names: dict[str, str] = None +) -> Group: + """ + Merges all instances of a peripheral group into as few variants as possible. + + Every shape is merged into the variant it shares the most elements with, so + that a shape that is compatible with several variants does not merely join + the first one. A shape that conflicts with every variant starts a new one. + + :param name: the name of the peripheral group, for example `TIM`. + :param instances: all instances of the group across all devices. + :param widening: whether a bit field that only grew into the reserved bits + above it is an optional extension instead of a conflict. + :param mode: `binary` locates elements by register address, `source` by + register name, and `similar` also tolerates differences in the + documentation, see `modm_data.svd2variants.merge.scope_of`. + :param names: the lookup table of generated feature names, see + `modm_data.svd2variants.annotate`. + :return: the group with its variants. + """ + group = Group(name, instances, mode=mode) + if ignored := IGNORED_REGISTERS.get(name): + group.ignored = sorted({r.name for i in instances for r in i.shape.registers if ignored.match(r.name)}) + for shape, shape_instances in _shapes(instances, ignored).items(): + member = Member(shape, ElementMap.from_shape(shape, mode), shape_instances) + best, shared, renames = None, -1, [] + for variant in group.variants: + difference = compare(variant.elements, member.elements, widening, mode == "similar") + if not difference.compatible: + continue + common = len(set(variant.elements.names) & set(member.elements.names)) + if common > shared: + best, shared, renames = variant, common, difference.renames + if best is None: + best = Variant(name) + group.variants.append(best) + renames = [] + best.add(member, renames) + + # A merged map collects aliases that are later evidence for a rename, so a + # conflict that kept two shapes apart may be gone once both variants are + # complete. Keep merging the smallest variant into its best match until no + # two variants are compatible anymore. + merged = True + while merged: + merged = False + for small in sorted(group.variants, key=lambda v: len(v.instances)): + best, shared, renames = None, -1, [] + for large in group.variants: + if large is small: + continue + difference = compare(large.elements, small.elements, widening, mode == "similar") + if not difference.compatible: + continue + common = len(set(large.elements.names) & set(small.elements.names)) + if common > shared: + best, shared, renames = large, common, difference.renames + if best is not None: + for member in small.members: + best.add(member, []) + best.renames += small.renames + renames + group.variants.remove(small) + merged = True + break + + for variant in group.variants: + variant.condense(mode, names) + group.variants.sort(key=lambda v: (-len(v.instances), -len(v.shapes))) + return group + + +def conflicts_between(variants: list[Variant], mode: str = "binary") -> dict[tuple[int, int], list[Conflict]]: + """:return: the conflicts between each pair of variants, which is why they are separate.""" + conflicts = {} + for left in range(len(variants)): + for right in range(left + 1, len(variants)): + difference = compare(variants[left].elements, variants[right].elements, similar=mode == "similar") + conflicts[(left, right)] = difference.conflicts + return conflicts diff --git a/src/modm_data/svd2variants/annotate.py b/src/modm_data/svd2variants/annotate.py new file mode 100644 index 0000000..f906d00 --- /dev/null +++ b/src/modm_data/svd2variants/annotate.py @@ -0,0 +1,316 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Naming Features with a Language Model + +The heuristic in `modm_data.svd2variants.naming` can only name a feature after +what its elements share lexically, which misses the semantics: the seventeen +bit fields of the first capture/compare channel of a timer share no name and no +description, but they are obviously "capture/compare channel 1". + +This module asks a small language model to name them instead and stores the +answers in a lookup table. The table is keyed by the *content* of the feature, +so it is stable across runs, reviewable as a diff, and only features that +actually changed need to be asked again. + +Names come from the first of these that works, so that a build never depends on +a model being there: + +1. the lookup table of a previous run, which the homepage and the CI download, +2. a local model served by LM Studio, see `local_endpoint`, or the `claude` + command line tool, +3. the heuristic of `modm_data.svd2variants.naming`. + +The prompt below is the result of measuring variants against each other on 91 +features of 12 peripheral groups. Keeping the numbers of similar elements as a +range and asking for a different name per feature both helped, while naming the +vague words to avoid produced more of them, and asking the model to first +explain what the elements have in common made the names worse. +""" + +import re +import json +import logging +import subprocess +import urllib.request +from hashlib import sha1 +from pathlib import Path +from multiprocessing.pool import ThreadPool + +from .naming import Element + +LOGGER = logging.getLogger(__name__) + +MODEL = "claude-haiku-4-5-20251001" +"""The model asked through the `claude` command line tool.""" +LOCAL_MODEL = "gemma-4-26b-a4b-it-qat-mlx" +"""The model asked through a local server, which LM Studio loads on demand.""" +LOCAL_ENDPOINT = "http://localhost:1234/v1" +"""Where LM Studio serves its OpenAI compatible API by default.""" +_MAX_ELEMENTS = 24 +_MAX_FEATURES = 16 + +_INSTRUCTIONS = """\ +You name the optional features of STM32 peripherals. Each block below lists registers and bit \ +fields of the STM32 {group} peripheral that exist on exactly the same set of devices, so together \ +they form one optional feature. + +Name each feature with a short noun phrase: +- 2 to 6 words in sentence case, without a trailing period. +- Say what the feature does, not which register it is in. Never use the word "register". +- Summarize many similar elements instead of listing them, but keep their numbers as a + range, e.g. "Lines 28 to 30 interrupt masking" or "Channels 3 and 4 DMA requests". +- Consider all elements, not only the first ones. +- Do not repeat the peripheral name {group}. +- Give every feature in the reply a different name, since they are different features. + +For example, these features of other peripherals: + +## a (5 elements in 4 registers) + CCMR1.CC1SEL — Capture/compare 1 selection + CCMR1.IC1PSC — Input capture 1 prescaler + CCR1.CCR1 — Capture/compare 1 value + DIER.CC1IE — Capture/compare 1 interrupt enable + SR.CC1IF — Capture/compare 1 interrupt flag + +## b (5 elements in 4 registers) + CR.WUTE — Wakeup timer enable + CR.WUTIE — Wakeup timer interrupt enable + WUTR.WUT — Wakeup auto-reload value + SR.WUTF — Wakeup timer flag + SCR.CWUTF — Clear wakeup timer flag + +## c (5 elements in 2 registers) + CR1.SMBHEN — SMBus host address enable + CR1.SMBDEN — SMBus device default address enable + CR1.ALERTEN — SMBus alert enable + CR1.PECEN — PEC enable + TIMEOUTR.TIMEOUTA — Bus timeout A + +## d (18 elements in 3 registers) + RXF0C.F0SA — Rx FIFO 0 start address + RXF0S.F0FL — Rx FIFO 0 fill level + IE.RF0NE — Rx FIFO 0 new message interrupt enable + RXF0C.F0S — Rx FIFO 0 size + ... and 14 more similar + +## e (2 elements in 2 registers) + POL.POL — Programmable polynomial + CR.POLYSIZE — Polynomial size + +are named: +{{"a": "Capture/compare channel 1", "b": "Wakeup timer", "c": "SMBus support", "d": "Receive FIFO 0", \ +"e": "Programmable polynomial"}} + +Reply with ONLY a JSON object mapping each id below to its name. +""" + + +def feature_key(elements: list[Element]) -> str: + """ + :param elements: the elements of one feature. + :return: a stable key for the content of the feature, so that a table entry + survives everything except a change to the feature itself. + """ + lines = sorted(f"{register}.{name}\t{description}" for _kind, register, name, description in elements) + return sha1("\n".join(lines).encode()).hexdigest()[:16] + + +def _sample(elements: list[Element]) -> list[Element]: + """ + Takes elements from all registers in turn, since small models tend to name a + feature after the first few elements, which are all in the same register. + """ + registers = {} + for element in elements: + registers.setdefault(element[1], []).append(element) + sample = [] + while len(sample) < min(_MAX_ELEMENTS, len(elements)): + for queue in registers.values(): + if queue and len(sample) < _MAX_ELEMENTS: + sample.append(queue.pop(0)) + return sample + + +def _block(index: int, elements: list[Element]) -> str: + registers = len({element[1] for element in elements}) + lines = [f"## {index} ({len(elements)} elements in {registers} registers)"] + for _kind, register, name, description in _sample(elements): + lines.append(f" {register}.{name}" + (f" — {description[:70]}" if description else "")) + if len(elements) > _MAX_ELEMENTS: + lines.append(f" ... and {len(elements) - _MAX_ELEMENTS} more similar") + return "\n".join(lines) + + +def _normalize(name: str) -> str | None: + """ + Brings a generated name into sentence case without a register suffix. + + :return: the name, or `None` if it is not a short noun phrase. + """ + name = " ".join(name.split()).strip(" .\"'") + name = re.sub(r"\s+registers?$", "", name, flags=re.IGNORECASE) + words = name.split(" ") + if not name or len(words) > 8 or len(name) > 70: + return None + + # Keep acronyms and mixed case words like IrDA, only lower title case words + def lower(word: str) -> str: + return "-".join(part.lower() if part[:1].isupper() and part[1:].islower() else part for part in word.split("-")) + + return " ".join([words[0][:1].upper() + words[0][1:]] + [lower(word) for word in words[1:]]) + + +def local_endpoint(endpoint: str = None, model: str = LOCAL_MODEL, timeout: float = 2) -> str | None: + """ + Looks for a local server that can name features, so that the names are only + generated where a model is actually available, see `modm_data.svd2variants`. + + :param endpoint: the URL to check, or the LM Studio default. + :param model: the model that the server must offer. + :return: the URL of the server, or `None` if it does not answer or lacks the model. + """ + url = endpoint or LOCAL_ENDPOINT + try: + with urllib.request.urlopen(url.rstrip("/") + "/models", timeout=timeout) as response: + models = {served.get("id") for served in json.loads(response.read()).get("data", [])} + except (OSError, ValueError): + return None + if model in models: + return url + LOGGER.info(f"{url} does not serve {model}, only {sorted(models)}") + return None + + +def _claude(prompt: str, model: str) -> str: + """Asks the model with the `claude` command line tool.""" + result = subprocess.run( + ["claude", "-p", "--model", model, "--allowed-tools", ""], + input=prompt, + capture_output=True, + text=True, + timeout=600, + ) + if result.returncode: + raise OSError(f"claude exited with {result.returncode}: {result.stderr[:200]}") + return result.stdout + + +def _openai(prompt: str, model: str, endpoint: str, reasoning: str = "none") -> str: + """ + Asks the model of an OpenAI compatible server, for example LM Studio at + `http://localhost:1234/v1`, Ollama or the llama.cpp server. + + :param reasoning: the reasoning effort. Reasoning models like Qwen3.5 think + for hundreds of tokens by default even for a short name, + and LM Studio only turns that off with `none`. + """ + request = urllib.request.Request( + endpoint.rstrip("/") + "/chat/completions", + data=json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": prompt}], + # Names should not change between runs + "temperature": 0, + "max_tokens": 2048, + **({"reasoning_effort": reasoning} if reasoning else {}), + } + ).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=900) as response: + answer = json.loads(response.read()) + return answer["choices"][0]["message"]["content"] or "" + + +def _ask( + group: str, batch: list[tuple[str, list[Element]]], model: str, endpoint: str = None, retry: bool = True +) -> dict[str, str]: + """Asks the model to name one batch of features of one peripheral group.""" + prompt = _INSTRUCTIONS.format(group=group) + "\n" + prompt += "\n\n".join(_block(index, elements) for index, (_key, elements) in enumerate(batch)) + try: + text = _openai(prompt, model, endpoint) if endpoint else _claude(prompt, model) + except (OSError, subprocess.TimeoutExpired, KeyError, IndexError, json.JSONDecodeError) as error: + LOGGER.warning(f"{group}: {error}") + return {} + + # Reasoning models may think out loud before answering + text = re.sub(r".*?", "", text, flags=re.DOTALL).strip() + if (start := text.find("{")) < 0 or (end := text.rfind("}")) < 0: + LOGGER.warning(f"{group}: no JSON in the answer: {text[:120]}") + return {} + try: + answers = json.loads(text[start : end + 1]) + except json.JSONDecodeError as error: + LOGGER.warning(f"{group}: {error}") + return {} + + names, missing = {}, [] + for index, (key, elements) in enumerate(batch): + answer = answers.get(str(index)) or answers.get(index) + if isinstance(answer, str) and (name := _normalize(answer)): + names[key] = name + else: + missing.append((key, elements)) + if not retry: + LOGGER.warning(f"{group}: no usable name for feature {index}: {answer!r}") + # Small models sometimes skip a few ids of a batch, which usually works on their own + if missing and retry: + names.update(_ask(group, missing, model, endpoint, retry=False)) + return names + + +def _batches(jobs: dict[str, list[tuple[str, list[Element]]]]) -> list[tuple[str, list]]: + batches = [] + for group, features in jobs.items(): + for start in range(0, len(features), _MAX_FEATURES): + batches.append((group, features[start : start + _MAX_FEATURES])) + return batches + + +def generate_names( + jobs: dict[str, list[tuple[str, list[Element]]]], + model: str = MODEL, + workers: int = 6, + progress=None, + endpoint: str = None, +) -> dict[str, str]: + """ + Asks the model to name every feature it is given. + + :param jobs: the features to name, as (key, elements) per peripheral group. + :param model: the model to ask. + :param workers: how many requests to run at the same time. + :param endpoint: the URL of an OpenAI compatible server instead of the + `claude` command line tool, e.g. `http://localhost:1234/v1`. + :param progress: called with the names of every finished batch, so that an + interrupted run, for example by a usage limit, keeps them. + :return: the generated names by feature key. + """ + batches = _batches(jobs) + names = {} + if not batches: + return names + with ThreadPool(workers) as pool: + results = pool.imap_unordered(lambda job: _ask(job[0], job[1], model, endpoint), batches) + for done, result in enumerate(results, 1): + names.update(result) + if progress is not None: + progress(result, done, len(batches)) + return names + + +def read_names(path: Path) -> dict[str, str]: + """:return: the lookup table of feature names, or an empty one.""" + if path is None or not Path(path).exists(): + return {} + return json.loads(Path(path).read_text()) + + +def write_names(names: dict[str, str], path: Path): + """Writes the lookup table sorted by key, so that it diffs cleanly.""" + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(json.dumps(dict(sorted(names.items())), indent=0, ensure_ascii=False) + "\n") diff --git a/src/modm_data/svd2variants/merge.py b/src/modm_data/svd2variants/merge.py new file mode 100644 index 0000000..59d16f2 --- /dev/null +++ b/src/modm_data/svd2variants/merge.py @@ -0,0 +1,527 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Comparing Register Maps + +Whether two register maps describe the same silicon cannot be decided by +comparing names alone, nor by comparing addresses alone: + +- ST renames registers and bit fields without changing the layout, most + visibly between the STM32H7x and STM32H7[RS]x devices. Comparing names alone + reports these as different implementations, which they are not. +- ST reuses layouts for entirely different registers. The STM32F1 `GPIO_CRL` + and the `GPIO_MODER` of every other device both start with two-bit fields at + the same positions, so comparing addresses alone merges the two GPIO + implementations into one, which they are not. + +Therefore both keys are compared, but they are *not* symmetric: + +- A name that appears at **two different locations** is a **conflict**: a + driver cannot be written against both maps, so they are different versions. + This is what separates the STM32F1 GPIO (`IDR` at `0x08`) from every other + GPIO (`IDR` at `0x10`). +- A location that carries **two different names** is a **rename**: the silicon + is the same and the names become aliases of each other. + +Names alone miss two more kinds of contradiction, since unrelated elements +never share a name: + +- Two bit fields with a different layout that **share bits** are a conflict, + unless they are alternates of each other in one map, see `overlaps`. This is + what separates the STM32L1 `COMP_CSR_OUTSEL[23:21]` from the STM32G0 + `COMP_CSR_BLANKING[24:20]`, which would otherwise each become an optional + feature of the other. +- A rename is only believable **inside the same register**, see `reuses`. The + STM32F373 `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` share a + bit, but the registers share neither a name nor a single bit field, so the + bit was reused, not renamed. + +Everything else is a *population* difference: an element that only exists in +one of the maps is an optional feature of the merged map. + +## Binary and Source Compatibility + +Many register maps differ only in *where* a register sits, while the register +itself is unchanged. Which of the two matters depends on how the hardware is +addressed, so the comparison runs in two modes, see `scope_of`: + +- `binary`: elements are located by the address of their register, so moving a + register to another offset is a conflict. This is what a driver that pokes + at fixed addresses needs. +- `source`: elements are located by the *name* of their register, so a + register may move as long as it keeps its name and its layout. This is what + code written against a CMSIS header needs, since the structure member + abstracts the offset away. A renamed register is a different register here, + and simply becomes an optional feature of the merged map. +- `similar`: like `source`, but conflicts that only exist in the + documentation are tolerated, see `documentation_only`. +""" + +import re +from dataclasses import dataclass, field as _field +from collections import defaultdict +from .model import Shape, LocationKey, NameKey + + +@dataclass +class Conflict: + """A name that is located differently in two register maps.""" + + name: NameKey + locations: set[LocationKey] + other: set[LocationKey] + + @property + def kind(self) -> str: + """ + - `widened`: a bit field that starts at the same position but is longer + in one of the maps, since ST tends to grow a field into the bits that + were reserved before, for example the `IWDG_PR` prescaler. + - `resized`: an element with the same name and address, but a different + size, which is not merely an extension of the shorter one. + - `moved`: an element with the same name at a different address. + """ + locations = self.locations | self.other + if self.name[0] == "F": + # Same register and same start bit, only the length of the field differs + return "widened" if len({location[1:3] for location in locations}) == 1 else "moved" + return "resized" if len({location[1] for location in locations}) == 1 else "moved" + + def __str__(self) -> str: + def location(keys): + return "/".join(sorted(_format_location(key) for key in keys)) + + return f"{_format_name(self.name)}: {self.kind} {location(self.locations)} != {location(self.other)}" + + +_INDEX = re.compile(r"_?\d+$") + + +@dataclass +class Overlap: + """ + Two bit fields with a different layout that share bits, without being + alternates of each other in any single register map. + """ + + location: LocationKey + names: set[NameKey] + other: LocationKey + others: set[NameKey] + + @property + def kind(self) -> str: + return "overlap" + + def __str__(self) -> str: + def label(location, names): + return f"{'/'.join(sorted(name[-1] for name in names))}[{_format_range(location)}]" + + return ( + f"{_format_scope(self.location[1])}: {label(self.location, self.names)} " + f"overlaps {label(self.other, self.others)}" + ) + + +def _format_range(location: LocationKey) -> str: + position, width = location[2], location[3] + return f"{position}" if width == 1 else f"{position + width - 1}:{position}" + + +def _same_field(names: set[NameKey], others: set[NameKey]) -> bool: + """ + Whether two names describe the same bit field, which either the name rules + already judge, or which one header describes bit by bit, e.g. `PLLM` and `PLLM_2`. + """ + for name in names: + for other in others: + if name[-1] == other[-1] or _INDEX.sub("", name[-1]) == other[-1] or _INDEX.sub("", other[-1]) == name[-1]: + return True + return False + + +def overlaps(left: "ElementMap", right: "ElementMap") -> list[Overlap]: + """ + Unrelated bit fields cannot share bits, but their names never collide, so + the name rules cannot see it. The STM32L1 `COMP_CSR_OUTSEL[23:21]` and the + STM32G0 `COMP_CSR_BLANKING[24:20]` are both at the same offset in the same + register, and without this check they merge into one register map, where + each becomes an optional feature of the other. + + :return: the new fields of `right` that overlap a differently laid out + field of `left`, which `right` does not have as an alternate. + """ + fields = defaultdict(list) + for location in left.names: + if location[0] == "F": + fields[location[1]].append(location) + found = [] + for location, names in right.names.items(): + if location[0] != "F" or location in left.names: + continue + _kind, scope, position, width = location + for other in fields.get(scope, ()): + if other[2] >= position + width or position >= other[2] + other[3]: + continue + # Alternates of the same register in one map may overlap, e.g. TIM_CCMR1 input and output + if other in right.names or _same_field(names, left.names[other]): + continue + found.append(Overlap(location, set(names), other, set(left.names[other]))) + return found + + +@dataclass +class Reuse: + """ + A location that is named differently in two register maps, but inside two + unrelated registers, so the names only coincide in the bits they use. + """ + + location: LocationKey + names: set[NameKey] + others: set[NameKey] + registers: set[str] + other_registers: set[str] + + @property + def kind(self) -> str: + return "reused" + + def __str__(self) -> str: + def label(names, registers): + return f"{'/'.join(sorted(registers))}.{'/'.join(sorted(name[-1] for name in names))}" + + where = _format_scope(self.location[1]) + if self.location[0] == "F": + where += f"[{_format_range(self.location)}]" + return f"{where}: {label(self.names, self.registers)} reused as {label(self.others, self.other_registers)}" + + +def _register_names(elements: "ElementMap") -> dict: + names = defaultdict(set) + for location, keys in elements.names.items(): + if location[0] == "R": + names[location[1]] |= {key[-1] for key in keys} + return names + + +def reuses(left: "ElementMap", right: "ElementMap", renames: list["Rename"]) -> tuple[list[Reuse], list["Rename"]]: + """ + A rename is only believable inside the same register. The STM32F373 + `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` sit at the same + bit, but the registers share neither a name nor a single bit field, so the + bit was reused for something else rather than renamed. Registers count as + the same if they share a name, or if at least one bit field has the same + name at the same location, e.g. `GPIO_AFRL` and `GPIO_AFR[0]`. + + :return: the renames that are actually reuses, and the remaining renames. + """ + left_registers, right_registers = _register_names(left), _register_names(right) + shared = { + location[1] + for location, names in right.names.items() + if location[0] == "F" and location in left.names and not left.names[location].isdisjoint(names) + } + # A register without bit fields on either side gives no evidence, e.g. an array whose bits were not named + left_fields = {location[1] for location in left.names if location[0] == "F"} + fields = left_fields & {location[1] for location in right.names if location[0] == "F"} + + reused, kept = [], [] + for rename in renames: + scope = rename.location[1] + same = not left_registers[scope].isdisjoint(right_registers[scope]) or scope in shared + if same or scope not in fields: + kept.append(rename) + else: + reused.append( + Reuse(rename.location, rename.names, rename.other, left_registers[scope], right_registers[scope]) + ) + return reused, kept + + +@dataclass +class Rename: + """A location that is named differently in two register maps.""" + + location: LocationKey + names: set[NameKey] + other: set[NameKey] + + def __str__(self) -> str: + def name(keys): + return "/".join(sorted(_format_name(key) for key in keys)) + + return f"{_format_location(self.location)}: {name(self.names)} = {name(self.other)}" + + +MODES = ("binary", "source", "similar") +"""The compatibility modes, see `scope_of` and `documentation_only`.""" + + +def scope_of(register, mode: str): + """ + The scope that an element is located in, which is what the two modes differ in: + + - `binary`: the offset of the register, so two maps are only compatible if + every register is at the same address. This is what a driver that pokes + at fixed addresses needs. + - `source`: the name of the register, so the offsets are free to move as + long as the register keeps its name and its layout. This is what code + written against a CMSIS header needs, since the header abstracts the + offsets away into structure members. + - `similar`: like `source`, but differences that only exist in the + documentation are no conflicts, see `documentation_only`. + """ + return register.offset if mode == "binary" else register.name + + +def register_location(register, mode: str) -> LocationKey: + return ("R", scope_of(register, mode), register.width, register.dim) + + +def register_name(register) -> NameKey: + return ("R", register.name) + + +def field_location(register, field, mode: str) -> LocationKey: + return ("F", scope_of(register, mode), field.position, field.width) + + +def field_name(register, field, mode: str) -> NameKey: + return ("F", scope_of(register, mode), field.name) + + +def _format_scope(scope) -> str: + return f"0x{scope:03x}" if isinstance(scope, int) else str(scope) + + +def _format_location(key: LocationKey) -> str: + if key[0] == "R": + return _format_scope(key[1]) + position, width = key[2], key[3] + return f"{_format_scope(key[1])}[{position if width == 1 else f'{position + width - 1}:{position}'}]" + + +def _format_name(key: NameKey) -> str: + return key[1] if key[0] == "R" else f"{_format_scope(key[1])}.{key[2]}" + + +class ElementMap: + """ + A bidirectional map between the locations and the names of the elements of + one or more register maps. Merged maps keep every alias of a location and + are therefore able to detect a conflict with any of them. + """ + + def __init__(self): + self.names: dict[LocationKey, set[NameKey]] = defaultdict(set) + self.locations: dict[NameKey, set[LocationKey]] = defaultdict(set) + + @classmethod + def from_shape(cls, shape: Shape, mode: str = "binary") -> "ElementMap": + elements = cls() + for register in shape.registers: + elements.add(register_location(register, mode), register_name(register)) + for field in register.fields: + elements.add(field_location(register, field, mode), field_name(register, field, mode)) + return elements + + def add(self, location: LocationKey, name: NameKey): + self.names[location].add(name) + self.locations[name].add(location) + + def update(self, other: "ElementMap"): + for location, names in other.names.items(): + self.names[location] |= names + for name, locations in other.locations.items(): + self.locations[name] |= locations + + @property + def scopes(self) -> set: + """The offsets or the names of all registers, depending on the mode.""" + return {location[1] for location in self.names if location[0] == "R"} + + def compare(self, other: "ElementMap") -> tuple[list[Conflict], list[Rename]]: + """ + :return: the names that are located differently in `other` and the + locations that are named differently in `other`. + """ + conflicts = [ + Conflict(name, set(self.locations[name]), set(locations)) + for name, locations in other.locations.items() + if name in self.locations and self.locations[name].isdisjoint(locations) + ] + renames = [ + Rename(location, set(self.names[location]), set(names)) + for location, names in other.names.items() + if location in self.names and self.names[location].isdisjoint(names) + ] + return conflicts, renames + + +@dataclass +class Difference: + """The result of comparing two register maps.""" + + conflicts: list[Conflict | Overlap | Reuse] = _field(default_factory=list) + widened: list[Conflict] = _field(default_factory=list) + tolerated: list = _field(default_factory=list) + """Conflicts that only differ in the documentation, see `documentation_only`.""" + renames: list[Rename] = _field(default_factory=list) + only_left: set[LocationKey] = _field(default_factory=set) + only_right: set[LocationKey] = _field(default_factory=set) + disjoint: bool = False + + @property + def compatible(self) -> bool: + """Whether both register maps can be merged into one.""" + return not self.conflicts and not self.disjoint + + @property + def identical(self) -> bool: + """Whether both register maps describe the same elements at the same locations.""" + return self.compatible and not self.only_left and not self.only_right and not self.renames + + @property + def relation(self) -> str: + if not self.compatible: + return "incompatible" + if not self.only_left and not self.only_right and not self.widened: + return "renamed" if self.renames else "identical" + if not self.only_right: + return "superset" + if not self.only_left: + return "subset" + return "overlapping" + + +def _bits(location: LocationKey) -> set[int]: + return set(range(location[2], location[2] + location[3])) + + +def _plain(name: str) -> str: + """The name without indices, placeholders and separators, e.g. `COMPBLANKING` for `COMPxBLANKING`.""" + return re.sub(r"[\d_]|x(?=[A-Z])", "", name).upper() + + +def _related(names: set[NameKey], others: set[NameKey], width: int) -> bool: + """ + Whether two bit field names are the same without their indices, e.g. `EM_12` + and `EM47`, one contains the other, e.g. `BLANKING` and `COMPxBLANKING`, or + they start the same, e.g. `FPU_IE` and `FPU_IOIE`. + + Different indices usually mean different instances of a bit field, e.g. the + `LPGPIO_MODER_MOD1[1]` of pin 1 inside the `GPIO_MODER_MODE0[1:0]` of pin 0, + unless the wider bit field is a mask of many, e.g. `EXTI_EMR2_EM[31:15]`. + + :param width: the width of the wider bit field. + """ + for name in names: + for other in others: + indices = [re.findall(r"\d+", key[-1]) for key in (name, other)] + if all(indices) and indices[0] != indices[1] and width < 4: + continue + short, long = sorted((_plain(name[-1]), _plain(other[-1])), key=len) + if short and (short == long or (len(short) >= 4 and short in long) or long.startswith(short[:3])): + return True + return False + + +def documentation_only(conflicts: list) -> tuple[list, list]: + """ + Separates the conflicts that are only differences in how a register is + documented from the ones that are differences in the silicon: + + - A register declared with another width, e.g. `USART_RDR` as `uint16_t` + in the STM32F0 header and as `uint32_t` everywhere else. + - A bit field with the same name over overlapping bits, e.g. a mask of many + bit fields that covers a different range. + - A bit field described as smaller bit fields with a related name, e.g. + `COMP_CSR_BLANKING` and `COMP_CSR_COMPxBLANKING`. + - A bit field of at least four bits described as smaller bit fields that + tile one of its edges, if they are a value split into at least two parts + of at least four bits, e.g. `USART_BRR_BRR` as `USART_BRR_DIV_MANTISSA` + and `USART_BRR_DIV_FRACTION`, or flags at the edge of a value that take + up at most a quarter of it, e.g. the `TIM_CNT_UIFCPY` flag in the top bit + of `TIM_CNT_CNT`. + + Unrelated bit fields nest by coincidence surprisingly often, which is not + tolerated: small ones, e.g. `DBGMCU_CR_TRACE_MODE[7:6]` and + `DBGMCU_CR_DBG_STOPD3[7]`, flags in another position, e.g. the STM32F3 + `SYSCFG_CFGR1_FPU_IE[31:26]` and the STM32F0 `SYSCFG_CFGR1_USART3_DMA_RMP[26]`, + and single wide bit fields, e.g. the STM32F4 `FLASH_OPTCR1_nWRP[27:16]` and + the STM32F7 `FLASH_OPTCR1_BOOT_ADD1[31:16]`. + + :return: the conflicts that differ in the silicon and the ones that don't. + """ + # All bit fields that nest inside a wider bit field, grouped by the wider one + nested, pairs = defaultdict(list), {} + for conflict in conflicts: + if isinstance(conflict, Overlap): + this, that = (conflict.location, conflict.names), (conflict.other, conflict.others) + inner, outer = sorted((this, that), key=lambda field: field[0][3]) + if _bits(inner[0]) <= _bits(outer[0]): + nested[outer[0]].append(inner[0]) + pairs[id(conflict)] = (inner, outer) + + def decomposes(outer: LocationKey) -> bool: + inners = nested[outer] + bits = set().union(*(_bits(inner) for inner in inners)) + low, high = min(bits), max(bits) + edge = bits == set(range(low, high + 1)) and (low == outer[2] or high == outer[2] + outer[3] - 1) + if outer[3] < 4 or not edge: + return False + parts = len(inners) >= 2 and all(inner[3] >= 4 for inner in inners) + flags = len(bits) * 4 <= outer[3] + return parts or flags + + silicon, documentation = [], [] + for conflict in conflicts: + if isinstance(conflict, Conflict): + locations = conflict.locations | conflict.other + same_bits = conflict.name[0] == "F" and bool(set.intersection(*(_bits(ll) for ll in locations))) + (documentation if conflict.kind == "resized" or same_bits else silicon).append(conflict) + elif (pair := pairs.get(id(conflict))) is not None: + inner, outer = pair + related = _related(inner[1], outer[1], outer[0][3]) + (documentation if related or decomposes(outer[0]) else silicon).append(conflict) + else: + silicon.append(conflict) + return silicon, documentation + + +def compare(left: ElementMap, right: ElementMap, widening: bool = True, similar: bool = False) -> Difference: + """ + Compares two register maps, see the module documentation for the rules. + + :param left: the register map to compare against, may be a merged map. + :param right: the register map to compare. + :param widening: whether a bit field that only grew into the reserved bits + above it is an optional extension instead of a conflict. + :param similar: whether differences in the documentation are tolerated, see + `documentation_only`. + :return: the differences between both maps. + """ + conflicts, renames = left.compare(right) + conflicts += overlaps(left, right) + reused, renames = reuses(left, right, renames) + conflicts += reused + widened = [] + if widening: + widened = [conflict for conflict in conflicts if conflict.kind == "widened"] + conflicts = [conflict for conflict in conflicts if conflict.kind != "widened"] + tolerated = [] + if similar: + conflicts, tolerated = documentation_only(conflicts) + # Two maps without a single common register describe unrelated hardware + disjoint = bool(left.names) and bool(right.names) and left.scopes.isdisjoint(right.scopes) + return Difference( + conflicts=conflicts, + widened=widened, + tolerated=tolerated, + renames=renames, + only_left=set(left.names) - set(right.names), + only_right=set(right.names) - set(left.names), + disjoint=disjoint, + ) diff --git a/src/modm_data/svd2variants/model.py b/src/modm_data/svd2variants/model.py new file mode 100644 index 0000000..edf2134 --- /dev/null +++ b/src/modm_data/svd2variants/model.py @@ -0,0 +1,143 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +The canonical description of a peripheral register map and its atomic elements. + +A register map is compared as a set of *elements*, where every element has both +a *location* and a *name*: + +- a register: `("R", offset, width, dim)` located, `("R", name)` named. +- a bit field inside that register: `("F", offset, position, width)` located, + `("F", offset, name)` named, so that the field names of a renamed register + still line up. + +Comparing both keys separately is what makes the difference between a register +map that was merely renamed and one that was actually changed, see +`modm_data.svd2variants.merge`. +""" + +import re +from dataclasses import dataclass, field as _field + +# Locations and names are compared as plain tuples for speed +LocationKey = tuple +NameKey = tuple + + +@dataclass(frozen=True, slots=True, order=True) +class Field: + """A bit field of a register.""" + + position: int + width: int + name: str + # Descriptions are documentation, not layout, and must not split a variant + description: str = _field(default="", compare=False) + + @property + def mask(self) -> int: + return ((1 << self.width) - 1) << self.position + + @property + def bit_range(self) -> str: + return f"{self.position}" if self.width == 1 else f"{self.position + self.width - 1}:{self.position}" + + +@dataclass(frozen=True, slots=True, order=True) +class Register: + """A register with its bit fields.""" + + offset: int + name: str + width: int = 4 + dim: int = 0 + fields: tuple[Field, ...] = () + description: str = _field(default="", compare=False) + # The CMSIS headers declare almost every register `__IO`, so the access is too + # sparse to tell implementations apart and is only carried along for display + access: str = _field(default="", compare=False) + + +@dataclass(frozen=True, slots=True) +class Shape: + """ + The register map of one peripheral instance, normalized so that two + instances with the same memory layout compare and hash equal. + + Registers sharing an offset are alternates of each other (unions in the + CMSIS header, for example `TIM_CCMR1` in output and input mode). They are + kept as separate registers, but their fields share one location scope, + since they describe the same physical register. + """ + + registers: tuple[Register, ...] + + @staticmethod + def from_peripheral(peripheral) -> "Shape": + """:param peripheral: a `modm_data.svd.Peripheral` tree node.""" + registers = [] + for register in peripheral.children: + fields = tuple( + sorted( + Field(f.position, f.width, f.name, getattr(f, "description", "") or "") for f in register.children + ) + ) + registers.append( + Register( + register.offset, + register.name, + register.width, + getattr(register, "dim", 0) or 0, + fields, + getattr(register, "description", "") or "", + getattr(register, "access", "") or "", + ) + ) + return Shape(tuple(sorted(registers))) + + def without(self, pattern: re.Pattern) -> "Shape": + """:return: the register map without the registers whose name matches the pattern.""" + return Shape(tuple(register for register in self.registers if not pattern.match(register.name))) + + @property + def offsets(self) -> set[int]: + return {register.offset for register in self.registers} + + def __len__(self) -> int: + return len(self.registers) + sum(len(register.fields) for register in self.registers) + + def __hash__(self) -> int: + return hash(self.registers) + + +IGNORED_REGISTERS: dict[str, re.Pattern] = { + # The option registers route device signals to the timer inputs, e.g. which + # comparator drives the break input or which pin is remapped to TI1. Their + # layout follows the device, not the timer, so they would split every timer + # implementation into one variant per device family. + "TIM": re.compile(r"^(OR\d?|AF\d|TISEL)$"), +} +"""Registers per peripheral group that are not part of the silicon implementation.""" + + +@dataclass +class Instance: + """One peripheral of one device.""" + + device: str + name: str + group: str + address: int + shape: Shape = _field(repr=False) + + +def group_name(peripheral) -> str: + """ + :param peripheral: a `modm_data.svd.Peripheral` tree node. + :return: the CMSIS structure type of the peripheral, for example `TIM` for + `TIM_TypeDef`, or the instance name without its index as fallback. + """ + if group := getattr(peripheral, "group", None): + return group + return re.sub(r"\d+", "", peripheral.name) or peripheral.name diff --git a/src/modm_data/svd2variants/naming.py b/src/modm_data/svd2variants/naming.py new file mode 100644 index 0000000..5155ee6 --- /dev/null +++ b/src/modm_data/svd2variants/naming.py @@ -0,0 +1,108 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Naming a Feature + +A feature is a set of elements that appear in exactly the same instances, which +is a mechanical definition and says nothing about what the feature *is*. Most +of them do have something in common though, since ST names and documents +related elements alike, so the name is derived from whatever the elements share: + +- a single element simply uses its own name and description, +- `RCC.DACEN`, `RCC.DACRST` and `RCC.DACLPEN` share the name prefix `DAC`, +- `EXTI.RT27` and `EXTI.FT27` share the description "trigger ... on line 27", +- `DMA.CCR4` … `DMA.CCR6` share the register name prefix `CCR`. + +About four out of five features are named this way. The rest are genuinely +heterogeneous, mostly large `RCC` and `DMA` features that enable a whole set of +unrelated peripherals or channels at once, and fall back to a plain count. +""" + +import re + +Element = tuple[str, str, str, str] +"""kind (`R` or `F`), register name, element name, description.""" + +_WORDS = re.compile(r"[A-Za-z0-9/]+") +# Words that carry no meaning on their own and make for a poor feature name +_FILLER = {"the", "and", "for", "bit", "bits", "register", "registers", "this", "not", "used"} + + +def _common_prefix(names: list[str]) -> str: + head, tail = min(names), max(names) + length = 0 + while length < len(head) and length < len(tail) and head[length] == tail[length]: + length += 1 + return head[:length].rstrip("_") + + +def _common_phrase(descriptions: list[str]) -> str: + """:return: the longest run of words that appears in every description.""" + words = [_WORDS.findall(description) for description in descriptions] + if not all(words): + return "" + shortest = min(words, key=len) + best = [] + for start in range(len(shortest)): + for end in range(len(shortest), start + len(best), -1): + phrase = [word.lower() for word in shortest[start:end]] + if all( + any( + phrase == [w.lower() for w in other[i : i + len(phrase)]] + for i in range(len(other) - len(phrase) + 1) + ) + for other in words + ): + best = shortest[start:end] + break + return " ".join(best).strip(" ,.:;-") + + +def name_feature(elements: list[Element], names: dict[str, str] = None) -> str: + """ + :param elements: the elements of one feature. + :param names: the lookup table of generated names, see + `modm_data.svd2variants.annotate`, which takes precedence. + :return: a short human readable name, see the module documentation. + """ + if not elements: + return "" + if names: + from .annotate import feature_key + + if generated := names.get(feature_key(elements)): + return generated + names = [element[2] for element in elements] + registers = sorted({element[1] for element in elements}) + descriptions = [element[3] for element in elements if element[3]] + + if len(elements) == 1: + kind, register, name, description = elements[0] + label = name if kind == "R" else f"{register}.{name}" + return f"{label} — {description}" if description else label + + if len(set(names)) == 1: + return f"{names[0]} in {len(registers)} registers" + + phrase = _common_phrase(descriptions) if len(descriptions) == len(elements) else "" + if len(phrase.split()) >= 2: + return phrase + + if len(prefix := _common_prefix(names)) >= 3: + return f"{prefix}*" + + if len(registers) == 1: + return f"{registers[0]}: {len(elements)} bit fields" + + if len(registers) > 1 and len(prefix := _common_prefix(registers)) >= 3: + return f"{prefix}* in {len(registers)} registers" + + if len(registers) <= 3: + return ", ".join(registers) + + # A single shared word is a weak name, but still better than a bare count + if len(phrase) >= 4 and phrase.lower() not in _FILLER: + return f"{phrase} in {len(registers)} registers" + + return f"{len(elements)} elements in {len(registers)} registers" diff --git a/src/modm_data/svd2variants/read.py b/src/modm_data/svd2variants/read.py new file mode 100644 index 0000000..ebe3151 --- /dev/null +++ b/src/modm_data/svd2variants/read.py @@ -0,0 +1,36 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +"""Reads the peripheral instances of many SVD files into one flat list.""" + +from pathlib import Path +from collections import defaultdict +from multiprocessing.pool import ThreadPool + +from ..svd import read_svd +from .model import Instance, Shape, group_name + + +def instances_of_svd(path: Path) -> list[Instance]: + """:return: every peripheral instance of one SVD file.""" + device = read_svd(path) + return [ + Instance( + device.name, peripheral.name, group_name(peripheral), peripheral.address, Shape.from_peripheral(peripheral) + ) + for peripheral in device.children + ] + + +def instances_of_svds(paths: list[Path]) -> dict[str, list[Instance]]: + """ + :param paths: the SVD files to read. + :return: every peripheral instance of every SVD file grouped by peripheral type. + """ + with ThreadPool() as pool: + results = pool.map(instances_of_svd, paths) + groups = defaultdict(list) + for instances in results: + for instance in instances: + groups[instance.group].append(instance) + return dict(sorted(groups.items())) diff --git a/src/modm_data/svd2variants/write.py b/src/modm_data/svd2variants/write.py new file mode 100644 index 0000000..ee377b3 --- /dev/null +++ b/src/modm_data/svd2variants/write.py @@ -0,0 +1,171 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +Serializes the peripheral variants into the JSON files that the Peripheral +Variant Explorer loads: one index of all groups and one file per group. + +Device names are shared per file and referenced by index, since the same few +hundred names would otherwise be repeated for every single variant, feature +and register map. +""" + +import json +from pathlib import Path +from collections import defaultdict +from .analyze import Group, Variant, conflicts_between +from .merge import ElementMap, register_location, field_location + + +def _names(elements: ElementMap, location: tuple) -> list[str]: + """:return: every name the element is known under, the most common one first.""" + names = elements.names.get(location, set()) + return sorted(name[-1] for name in names) + + +def _register_map(variant: Variant, features: dict[tuple, int], mode: str) -> list[dict]: + """The merged register map of the variant, with every element tagged by its feature.""" + registers = {} + descriptions = {} + offsets = defaultdict(set) + accesses = defaultdict(set) + for member in variant.members: + for register in member.shape.registers: + location = register_location(register, mode) + entry = registers.setdefault( + location, + { + "width": register.width, + "dim": register.dim, + "names": _names(variant.elements, location), + "feature": features.get(location, -1), + "fields": {}, + }, + ) + # In source mode the same register may sit at a different offset per instance + offsets[location].add(register.offset) + descriptions.setdefault(location, register.description) + if register.access: + accesses[location].add(register.access) + for field in register.fields: + flocation = field_location(register, field, mode) + entry["fields"].setdefault( + flocation, + { + "position": field.position, + "width": field.width, + "names": _names(variant.elements, flocation), + "feature": features.get(flocation, -1), + "description": field.description, + }, + ) + for location, entry in registers.items(): + entry["description"] = descriptions.get(location, "") + entry["offset"] = min(offsets[location]) + # Instances may disagree, e.g. a register that only some headers declare read-only + entry["access"] = sorted(accesses[location]) + if len(offsets[location]) > 1: + entry["offsets"] = sorted(offsets[location]) + entry["fields"] = sorted(entry.pop("fields").values(), key=lambda f: f["position"]) + return sorted(registers.values(), key=lambda r: (r["offset"], r["names"])) + + +def _variant(variant: Variant, index: int, devices: dict[str, int], conflicts: dict, mode: str) -> dict: + features = {location: index for index, feature in enumerate(variant.features) for location in feature.locations} + return { + "instances": len(variant.instances), + "devices": sorted(devices[device] for device in variant.devices), + "names": sorted(variant.names), + "members": [ + { + "instances": len(member.instances), + "devices": sorted(devices[device] for device in member.devices), + "names": sorted({instance.name for instance in member.instances}), + } + for member in variant.members + ], + "features": [ + { + "name": feature.name, + "elements": len(feature.locations), + "registers": len(feature.registers), + "instances": feature.instances, + "members": sorted(feature.shapes), + "devices": sorted( + {devices[device] for member in feature.shapes for device in variant.members[member].devices} + ), + } + for feature in variant.features + ], + "renames": sorted({str(rename) for rename in variant.renames}), + "conflicts": { + str(other): [str(conflict) for conflict in found] + for (left, other), found in conflicts.items() + if left == index and found + }, + "registers": _register_map(variant, features, mode), + } + + +def group_to_json(group: Group) -> dict: + """:return: the group with all of its variants as a JSON compatible dictionary.""" + devices = {device: index for index, device in enumerate(sorted(group.devices))} + conflicts = conflicts_between(group.variants, group.mode) + return { + "name": group.name, + "mode": group.mode, + "ignored": group.ignored, + "instances": len(group.instances), + "devices": sorted(devices), + "variants": [ + _variant(variant, index, devices, conflicts, group.mode) for index, variant in enumerate(group.variants) + ], + } + + +def index_to_json(groups: list[Group]) -> dict: + """:return: the summary of all groups as a JSON compatible dictionary.""" + return { + "groups": [ + { + "name": group.name, + "ignored": group.ignored, + "instances": len(group.instances), + "devices": len(group.devices), + "shapes": group.shapes, + "variants": [ + { + "instances": len(variant.instances), + "devices": len(variant.devices), + "members": len(variant.members), + "registers": len({location for location in variant.elements.names if location[0] == "R"}), + "core": len(variant.core), + "features": len(variant.features), + } + for variant in group.variants + ], + } + for group in groups + ], + "instances": sum(len(group.instances) for group in groups), + "shapes": sum(group.shapes for group in groups), + "variants": sum(len(group.variants) for group in groups), + } + + +def write_variants(modes: dict[str, list[Group]], path: Path): + """ + Writes `variants.json` with the summary of every group in every mode and + `/.json` with the full register maps of every group. + + :param modes: the analyzed peripheral groups per compatibility mode. + :param path: the output folder. + """ + index = {} + for mode, groups in modes.items(): + folder = path / mode + folder.mkdir(parents=True, exist_ok=True) + for group in groups: + (folder / f"{group.name}.json").write_text(json.dumps(group_to_json(group), separators=(",", ":")) + "\n") + index[mode] = index_to_json(groups) + (path / "variants.json").write_text(json.dumps({"modes": index}, indent=0) + "\n") diff --git a/tools/make/common.mk b/tools/make/common.mk index 1957bed..f98995b 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -62,6 +62,12 @@ build-homepage: @python3 -m modm_data.header2svd.stmicro --all --output docs/src/svd/ @curl -fsSL --create-dirs -o docs/src/svd/index.html \ https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/index.html + @curl -fsSL --create-dirs -o ext/cache/svd2variants/feature-names-gemma.json \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/feature-names-gemma.json || true + @python3 -m modm_data.svd2variants --all --input docs/src/svd/ \ + --names ext/cache/svd2variants/feature-names-gemma.json --output docs/src/variants/ + @curl -fsSL --create-dirs -o docs/src/variants/index.html \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/variants.html @(cd docs && mkdocs build)