From 6468f1d59ae80105c8f12ec9344e67f1437e279c Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Thu, 17 Sep 2026 21:39:41 +0200 Subject: [PATCH 1/5] [svd] Preserve the peripheral type, access and enumerated values The reader dropped everything except names, offsets and bit ranges, and the writer had no place for the CMSIS structure type of a peripheral, which is the only reliable way to tell which instances are the same kind of hardware. Write it as the standard groupName element together with the register access, and read back the descriptions, alternates, array dimensions, enumerated values and interrupts. --- src/modm_data/svd/read.py | 46 ++++++++++++++++++++++++++++++++------ src/modm_data/svd/write.py | 4 ++++ 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/modm_data/svd/read.py b/src/modm_data/svd/read.py index 3cdd781..f3c544d 100644 --- a/src/modm_data/svd/read.py +++ b/src/modm_data/svd/read.py @@ -3,7 +3,7 @@ import re from lxml import etree -from .model import Device, Peripheral, Register, BitField +from .model import Device, Peripheral, Register, BitField, EnumeratedValue def _int(node, tag, default=None): @@ -12,6 +12,12 @@ def _int(node, tag, default=None): return int(text.strip().replace("#", "0b"), 0) +def _text(node, tag) -> str | None: + if (text := node.findtext(tag)) is None: + return None + return " ".join(text.split()) + + def _bit_range(node) -> tuple[int, int]: if (offset := _int(node, "bitOffset")) is not None: return offset, _int(node, "bitWidth", 1) @@ -21,6 +27,17 @@ def _bit_range(node) -> tuple[int, int]: return lsb, msb - lsb + 1 +def _add_bit_fields(rnode, register): + for fnode in rnode.iter("field"): + field = BitField(fnode.findtext("name"), *_bit_range(fnode), parent=register) + field.description = _text(fnode, "description") + for vnode in fnode.iter("enumeratedValue"): + if (value := _text(vnode, "value")) is None: + continue + enum = EnumeratedValue(vnode.findtext("name"), value.replace("#", "0b"), parent=field) + enum.description = _text(vnode, "description") + + def read_svd(path) -> Device: """ Reads the peripherals, registers and bit fields of a CMSIS-SVD file. @@ -37,10 +54,19 @@ def read_svd(path) -> Device: for node in root.iter("peripheral"): name = node.findtext("name") registers[name] = node.find("registers") - peripherals.append((name, node.get("derivedFrom"), _int(node, "baseAddress"))) + peripherals.append(node) - for name, derived_from, address in peripherals: - peripheral = Peripheral(name, derived_from, address, parent=device) + for node in peripherals: + name = node.findtext("name") + derived_from = node.get("derivedFrom") + peripheral = Peripheral(name, _text(node, "groupName"), _int(node, "baseAddress"), parent=device) + peripheral.group = _text(node, "groupName") + peripheral.description = _text(node, "description") + peripheral.alternate = _text(node, "alternatePeripheral") + peripheral.interrupts = [ + (inode.findtext("name"), _int(inode, "value"), _text(inode, "description")) + for inode in node.iter("interrupt") + ] if derived_from: peripheral.derived_from = derived_from if (nodes := registers[name]) is None and derived_from: @@ -49,8 +75,14 @@ def read_svd(path) -> Device: continue for rnode in nodes.iter("register"): size = _int(rnode, "size", 32) // 8 - register = Register(rnode.findtext("name"), _int(rnode, "addressOffset"), size, parent=peripheral) - for fnode in rnode.iter("field"): - BitField(fnode.findtext("name"), *_bit_range(fnode), parent=register) + rname = rnode.findtext("name") + register = Register(rname.replace("[%s]", ""), _int(rnode, "addressOffset"), size, parent=peripheral) + register.description = _text(rnode, "description") + register.alternate = _text(rnode, "alternateRegister") + register.access = _text(rnode, "access") + if (dim := _int(rnode, "dim")) is not None: + register.dim = dim + register.dim_increment = _int(rnode, "dimIncrement", size) + _add_bit_fields(rnode, register) return device diff --git a/src/modm_data/svd/write.py b/src/modm_data/svd/write.py index 9fd0b00..a46e8e3 100644 --- a/src/modm_data/svd/write.py +++ b/src/modm_data/svd/write.py @@ -33,6 +33,8 @@ def _format_peripheral(xmlnode, treenode): _add_element(peripheral, "name", treenode.name) if description := getattr(treenode, "description", None): _add_element(peripheral, "description", description) + if group := getattr(treenode, "group", None): + _add_element(peripheral, "groupName", group) if alternate := getattr(treenode, "alternate", None): _add_element(peripheral, "alternatePeripheral", alternate) _add_element(peripheral, "baseAddress", hex(treenode.address)) @@ -62,6 +64,8 @@ def _format_register(xmlnode, treenode): _add_element(register, "alternateRegister", alternate) _add_element(register, "addressOffset", hex(treenode.offset)) _add_element(register, "size", hex(treenode.width * 8)) + if access := getattr(treenode, "access", None): + _add_element(register, "access", access) if treenode.children: return _add_element(register, "fields") else: From 740ebaee1119be4da070e323f8257bfec05c8419 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Thu, 17 Sep 2026 21:39:41 +0200 Subject: [PATCH 2/5] [header2svd] Fix the values of non-contiguous bit fields and add the access The values of a bit field were only removed if the bit field was contiguous, so the value macros of CRYP_CR_ALGOMODE (bits 5:3 and 19) replaced the bit field itself and were spread over three alternate registers. Match them against the name of the whole bit field instead, which changes CRYP_CR on 27 devices and FLASH_OPTR_nRST_MODE on the STM32WB1x, and nothing else of the 1.7 million bit fields. Members declared __I or __IM expand to volatile const and are read-only, everything else is read-write. The __O qualifier expands to the same volatile as __IO, so write-only registers cannot be told apart, but no ST header uses it: 136633 members are __IO, 711 __I and 207 __IM. --- src/modm_data/header2svd/extract.py | 9 +++++++-- src/modm_data/header2svd/stmicro/memory_map.py | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/modm_data/header2svd/extract.py b/src/modm_data/header2svd/extract.py index 829c59b..c60708a 100644 --- a/src/modm_data/header2svd/extract.py +++ b/src/modm_data/header2svd/extract.py @@ -37,7 +37,7 @@ _CMSIS_PATH = ext_path("arm/cmsis/CMSIS/Core/Include") _CACHE_PATH = cache_path("cmsis/header2svd") -_VERSION = 2 +_VERSION = 3 @dataclass @@ -48,6 +48,10 @@ class Member: """Type name of the member, or `pointer` for pointer members.""" length: str | None """The array length expression or `None` if the member is not an array.""" + access: str = "read-write" + """`read-only` for members declared with `__I` or `__IM`, which expand to + `volatile const`. The `__O` qualifier expands to the same `volatile` as `__IO`, + so write-only members cannot be told apart, but ST does not use it.""" @dataclass @@ -152,6 +156,7 @@ def members(cls, name): if isinstance(mtype, ctypes.Array): length = "".join(token.value for token in mtype.size.tokens) mtype = mtype.array_of + access = "read-only" if getattr(mtype, "const", False) else "read-write" if isinstance(mtype, (ctypes.Pointer, ctypes.FunctionType)) or not hasattr(mtype, "typename"): if member.name: result.append(Member(member.name, "pointer", length)) @@ -171,7 +176,7 @@ def members(cls, name): result.append(Member(member.name, subtype, length)) continue if member.name: - result.append(Member(member.name, segment.name, length)) + result.append(Member(member.name, segment.name, length, access)) # Anonymous unions and structures without a member name for cid, inner_cls in inner.items(): if cid not in named: diff --git a/src/modm_data/header2svd/stmicro/memory_map.py b/src/modm_data/header2svd/stmicro/memory_map.py index 58e9621..0d3505f 100644 --- a/src/modm_data/header2svd/stmicro/memory_map.py +++ b/src/modm_data/header2svd/stmicro/memory_map.py @@ -509,12 +509,14 @@ def _bit_fields(register: _Register, fields: dict[str, str], bitfields: _BitFiel for position, width, index in bitfields.fields[macro]: split = len(bitfields.fields[macro]) > 1 fname = f"{name}_{index}" if index else name - candidates.append((fname, position, width, macro, split)) + candidates.append((fname, position, width, macro, split, name)) def bits(position, width): return set(range(position, position + width)) - # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0 + # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0. The parent + # may be one part of a non-contiguous bit field, e.g. CRYP_CR_ALGOMODE_AES_KEY in bits 5:3 of CRYP_CR_ALGOMODE, + # which also uses bit 19, otherwise the values replace the bit field itself. result = [] for candidate in candidates: cbits = bits(*candidate[1:3]) @@ -522,7 +524,10 @@ def bits(position, width): ( c for c in candidates - if c is not candidate and candidate[0].startswith(c[0] + "_") and cbits <= bits(*c[1:3]) and not c[4] + if c is not candidate + and candidate[0] != c[0] + and candidate[0].startswith(c[5] + "_") + and cbits <= bits(*c[1:3]) ), None, ) @@ -547,7 +552,7 @@ def bits(position, width): # Prefer non-alias and non-split bit fields in the order of definition result.sort(key=lambda c: (c[3] in bitfields.alias, c[4], bitfields.order.get(c[3], 0), c[1])) used, alternate = {}, [] - for fname, position, width, macro, split in result: + for fname, position, width, macro, split, _name in result: overlap = next((used[b] for b in bits(position, width) if b in used), None) if overlap is not None and fname not in register.fields: # A narrow field at the edge of a much wider field, e.g. TIM_CNT_UIFCPY in TIM_CNT_CNT @@ -769,6 +774,7 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> interrupts = _interrupts(data, {re.sub(r"_NS$", "", pname) for pname in peripherals}, report) device = Device(name or data.header.stem, compatible=data.defines[:1]) + access = {(typedef, member.name): member.access for typedef, members in data.structs.items() for member in members} signatures, addresses = {}, {} for pname, (address, ptype, registers) in sorted(peripherals.items(), key=lambda p: (p[1][0], p[0])): # Registers of sub-instances with colliding names are prefixed with the sub-instance name @@ -791,6 +797,8 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> # The non-secure instances use the plain name peripheral = Peripheral(re.sub(r"_NS$", "", pname), ptype, address, parent=device) peripheral.description = data.type_descriptions.get(ptype, "") + if ptype: + peripheral.group = re.sub(r"_?TypeDef$", "", ptype) peripheral.interrupts = interrupts.get(peripheral.name, []) if (alternate := addresses.setdefault(address, peripheral.name)) != peripheral.name: peripheral.alternate = alternate @@ -800,6 +808,7 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> offsets = {} for register in registers: treg = Register(register.name, register.offset, register.size, parent=peripheral) + treg.access = access.get((register.type, register.member), "read-write") treg.description = data.member_descriptions.get((register.type, register.member), "") if register.dim: treg.dim = register.dim From 05e0bfafadf2d036555de3797f128c424332607e Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Thu, 17 Sep 2026 21:39:59 +0200 Subject: [PATCH 3/5] [svd2variants] Merge the register maps of all devices into variants Every STM32 device has its own register map, but the peripherals inside them are reused across the whole device range. This merges the register maps of all 22001 peripheral instances of all 287 devices into as few variants as possible, where each variant is one silicon implementation described by a core that every instance has and the optional features that only some have. A feature is every element that appears in exactly the same instances, since those are enabled and disabled together. Neither names nor addresses alone can decide whether two register maps are the same silicon, so both are compared and not symmetrically: a name at two locations is a conflict, a location with two names is a rename. Since unrelated elements never share a name, differently laid out bit fields that share bits are a conflict too, and a rename only counts inside the same register. Otherwise the STM32L1, STM32L0, STM32G0 and STM32H503 comparators all merge into one variant of unrelated bit fields. The elements are located by register address in binary mode and by register name in source mode, which is what code written against a CMSIS header needs, while similar mode also tolerates differences that only exist in the documentation, like USART_BRR against DIV_MANTISSA and DIV_FRACTION. The timer option registers are left out entirely, since they route device signals and would split the timers per family. 1808 distinct register maps become 387 binary, 319 source and 282 similar variants. Their features are named by a lookup table, a local model served by LM Studio, or a heuristic, whichever is available first. --- src/modm_data/svd2variants/__init__.py | 46 +++ src/modm_data/svd2variants/__main__.py | 203 ++++++++++ src/modm_data/svd2variants/analyze.py | 238 +++++++++++ src/modm_data/svd2variants/annotate.py | 316 +++++++++++++++ src/modm_data/svd2variants/merge.py | 527 +++++++++++++++++++++++++ src/modm_data/svd2variants/model.py | 143 +++++++ src/modm_data/svd2variants/naming.py | 108 +++++ src/modm_data/svd2variants/read.py | 36 ++ src/modm_data/svd2variants/write.py | 171 ++++++++ 9 files changed, 1788 insertions(+) create mode 100644 src/modm_data/svd2variants/__init__.py create mode 100644 src/modm_data/svd2variants/__main__.py create mode 100644 src/modm_data/svd2variants/analyze.py create mode 100644 src/modm_data/svd2variants/annotate.py create mode 100644 src/modm_data/svd2variants/merge.py create mode 100644 src/modm_data/svd2variants/model.py create mode 100644 src/modm_data/svd2variants/naming.py create mode 100644 src/modm_data/svd2variants/read.py create mode 100644 src/modm_data/svd2variants/write.py diff --git a/src/modm_data/svd2variants/__init__.py b/src/modm_data/svd2variants/__init__.py new file mode 100644 index 0000000..6594498 --- /dev/null +++ b/src/modm_data/svd2variants/__init__.py @@ -0,0 +1,46 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Peripheral Variants from SVD Files + +Every STM32 device has its own register map, but the peripherals inside them +are reused across the entire device range. This module answers how many +*different* implementations of a peripheral actually exist by merging the +register maps of all peripheral instances of all devices into as few variants +as possible. + +Each variant is one silicon implementation, described by a common core that +every instance implements and a set of optional features that only some +instances implement. Two variants of the same peripheral cannot be merged, +since they contradict each other, see `modm_data.svd2variants.merge` for what +exactly makes two register maps contradict. +""" + +from .model import Field, Register, Shape, Instance, group_name +from .merge import MODES, ElementMap, Conflict, Rename, Difference, compare, scope_of +from .analyze import Feature, Member, Variant, Group, variants_of, conflicts_between +from .read import instances_of_svd, instances_of_svds + +__all__ = [ + "Field", + "Register", + "Shape", + "Instance", + "group_name", + "MODES", + "ElementMap", + "scope_of", + "Conflict", + "Rename", + "Difference", + "compare", + "Feature", + "Member", + "Variant", + "Group", + "variants_of", + "conflicts_between", + "instances_of_svd", + "instances_of_svds", +] diff --git a/src/modm_data/svd2variants/__main__.py b/src/modm_data/svd2variants/__main__.py new file mode 100644 index 0000000..31530b4 --- /dev/null +++ b/src/modm_data/svd2variants/__main__.py @@ -0,0 +1,203 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +import re +import logging +import argparse +from pathlib import Path +from collections import defaultdict + +from modm_data.svd2variants import instances_of_svds, variants_of, conflicts_between +from modm_data.svd2variants.merge import MODES +from modm_data.svd2variants.write import write_variants +from modm_data.svd2variants.annotate import ( + LOCAL_MODEL, + MODEL, + feature_key, + generate_names, + local_endpoint, + read_names, + write_names, +) +from modm_data.utils import ext_path + +LOGGER = logging.getLogger("svd2variants") + + +def _report(group) -> str: + lines = [ + f"{group.name}: {len(group.variants)} {group.mode} compatible variants of {group.shapes} register maps", + "", + ] + for index, variant in enumerate(group.variants): + registers = {location for location in variant.elements.names if location[0] == "R"} + lines.append( + f"[{index}] {len(variant.instances)} instances on {len(variant.devices)} devices, " + f"{len(variant.members)} register maps, {len(registers)} registers, " + f"{len(variant.core)} core elements, {len(variant.features)} features" + ) + lines.append(f" instances: {' '.join(sorted(variant.names))}") + lines.append(f" devices: {' '.join(sorted(variant.devices))}") + for number, feature in enumerate(variant.features): + lines.append( + f" feature {number} ({feature.name}): {len(feature.locations)} elements in " + f"{len(feature.shapes)}/{len(variant.members)} register maps, max {feature.instances} instances" + ) + if variant.renames: + lines.append(f" renamed: {len(set(map(str, variant.renames)))}") + lines += [f" {rename}" for rename in sorted({str(r) for r in variant.renames})] + lines.append("") + for (left, right), conflicts in conflicts_between(group.variants, group.mode).items(): + lines.append(f"Conflicts between [{left}] and [{right}]: {len(conflicts)}") + lines += [f" {conflict}" for conflict in conflicts] + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser( + description="Merges the register maps of all peripheral instances into as few variants as possible." + ) + parser.add_argument( + "--group", + type=str, + default=[], + action="append", + help="Regex pattern of peripheral groups to analyze, e.g. GPIO.", + ) + parser.add_argument("--all", action="store_true", default=False, help="Analyze all peripheral groups.") + parser.add_argument( + "--input", + type=Path, + default=None, + help="Folder with the SVD files, defaults to ext/stmicro/svd.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Folder for the variants.json and the per group JSON files.", + ) + parser.add_argument( + "--strict", + action="store_true", + default=False, + help="Treat a bit field that only grew into the reserved bits above it as a conflict.", + ) + parser.add_argument( + "--mode", + choices=[*MODES, "all"], + default="all", + help="Locate elements by register address (binary), by register name (source), or by register name while tolerating differences in the documentation (similar).", + ) + parser.add_argument( + "--names", + type=Path, + default=None, + help="JSON lookup table of generated feature names, which takes precedence over the heuristic.", + ) + parser.add_argument( + "--no-generate", + action="store_true", + default=False, + help="Only use the names of --names, even if a model is available to name the rest.", + ) + parser.add_argument( + "--claude", + action="store_true", + default=False, + help="Name the missing features with the claude command line tool instead of a local server.", + ) + parser.add_argument("--model", type=str, default=None, help="The model that names the features.") + parser.add_argument( + "--endpoint", + type=str, + default=None, + help=f"URL of an OpenAI compatible server, defaults to LM Studio serving {LOCAL_MODEL}.", + ) + parser.add_argument( + "--workers", + type=int, + default=None, + help="Number of concurrent requests, defaults to 6 for claude and 1 for a local server.", + ) + parser.add_argument("-v", "--verbose", action="count", default=0) + args = parser.parse_args() + logging.basicConfig(level=[logging.WARNING, logging.INFO, logging.DEBUG][min(args.verbose, 2)]) + + paths = sorted((args.input or ext_path("stmicro/svd")).glob("*.svd")) + if not paths: + print("No SVD files found!") + return False + instances = instances_of_svds(paths) + names = [n for n in instances if args.all or any(re.match(p, n) for p in args.group)] + if not names: + print("No matching peripheral groups found!") + return False + + table = read_names(args.names) + modes = { + mode: [variants_of(name, instances[name], widening=not args.strict, mode=mode, names=table) for name in names] + for mode in (MODES if args.mode == "all" else [args.mode]) + } + + if not args.no_generate: + jobs = defaultdict(list) + seen = set() + for groups in modes.values(): + for group in groups: + for variant in group.variants: + for feature in variant.features: + key = feature_key(feature.elements) + if key not in table and key not in seen: + seen.add(key) + jobs[group.name].append((key, feature.elements)) + # The table of a previous run first, then a local model, then the heuristic + endpoint = None if args.claude else local_endpoint(args.endpoint, args.model or LOCAL_MODEL) + model = args.model or (MODEL if args.claude else LOCAL_MODEL) + if seen and (endpoint or args.claude): + print(f"Naming {len(seen)} features of {len(jobs)} peripheral groups with {model}...") + + def progress(result, done, total): + table.update(result) + if args.names: + write_names(table, args.names) + print(f" batch {done}/{total}: {len(result)} names", flush=True) + + workers = args.workers or (6 if args.claude else 1) + generated = generate_names(jobs, model, workers, progress, endpoint) + if args.names: + print(f"{len(table)} names in {args.names}") + # Name the features again, now that the table is filled + if generated: + modes = { + mode: [ + variants_of(name, instances[name], widening=not args.strict, mode=mode, names=table) + for name in names + ] + for mode in (MODES if args.mode == "all" else [args.mode]) + } + elif seen: + print(f"{len(seen)} features are named by the heuristic, no model available to name them.") + for mode, groups in modes.items(): + log_path = Path("log/stmicro/variants") / mode + log_path.mkdir(exist_ok=True, parents=True) + for group in groups: + (log_path / f"{group.name}.txt").write_text(_report(group)) + if args.output: + write_variants(modes, args.output) + + header = "".join(f"{mode + ' variants':>18}" for mode in modes) + print(f"{'group':24} {'instances':>9} {'maps':>6}{header}") + for group in sorted(next(iter(modes.values())), key=lambda g: (-len(g.variants), -g.shapes)): + counts = "".join( + f"{len(next(g for g in groups if g.name == group.name).variants):18}" for groups in modes.values() + ) + print(f"{group.name:24} {len(group.instances):9} {group.shapes:6}{counts}") + totals = "".join(f"{sum(len(g.variants) for g in groups):18}" for groups in modes.values()) + first = next(iter(modes.values())) + print(f"{'Total':24} {sum(len(g.instances) for g in first):9} {sum(g.shapes for g in first):6}{totals}") + return True + + +if __name__ == "__main__": + exit(0 if main() else 1) diff --git a/src/modm_data/svd2variants/analyze.py b/src/modm_data/svd2variants/analyze.py new file mode 100644 index 0000000..8ae047a --- /dev/null +++ b/src/modm_data/svd2variants/analyze.py @@ -0,0 +1,238 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Condensing Register Maps into Variants + +All instances of a peripheral group are first deduplicated into distinct +*shapes*, then greedily merged into *variants*: a variant is a set of shapes +that can be merged without a conflict, and therefore describes one silicon +implementation of the peripheral. + +The merged register map of a variant is split into a **core**, which every +instance of the variant implements, and a set of **features**, which are the +elements that only some instances implement. A feature is not a single +register or bit field, but every element that appears in exactly the same +instances, since those are enabled and disabled together, for example the +second half of the CAN filter banks or the clock enable bits of a peripheral +that is only bonded out on the larger packages. +""" + +import re +from dataclasses import dataclass, field as _field +from collections import defaultdict +from .model import Shape, Instance, LocationKey, IGNORED_REGISTERS +from .merge import ElementMap, Conflict, Rename, compare, register_location, field_location +from .naming import name_feature + + +@dataclass +class Feature: + """The elements of a variant that are implemented by the same instances.""" + + locations: set[LocationKey] + shapes: frozenset[int] = _field(repr=False) + instances: int = 0 + name: str = "" + elements: list = _field(default_factory=list, repr=False) + + @property + def registers(self) -> set[int]: + return {location[1] for location in self.locations if location[0] == "R"} + + +@dataclass +class Member: + """One distinct register map of a variant and every instance that has it.""" + + shape: Shape = _field(repr=False) + elements: ElementMap = _field(repr=False) + instances: list[Instance] = _field(default_factory=list, repr=False) + + @property + def devices(self) -> set[str]: + return {instance.device for instance in self.instances} + + +@dataclass +class Variant: + """One silicon implementation of a peripheral group.""" + + group: str + members: list["Member"] = _field(default_factory=list, repr=False) + elements: ElementMap = _field(default_factory=ElementMap, repr=False) + renames: list[Rename] = _field(default_factory=list, repr=False) + core: set[LocationKey] = _field(default_factory=set, repr=False) + features: list[Feature] = _field(default_factory=list, repr=False) + + @property + def shapes(self) -> list[Shape]: + return [member.shape for member in self.members] + + @property + def instances(self) -> list[Instance]: + return [instance for member in self.members for instance in member.instances] + + @property + def devices(self) -> set[str]: + return {instance.device for member in self.members for instance in member.instances} + + @property + def names(self) -> set[str]: + return {instance.name for member in self.members for instance in member.instances} + + def add(self, member: "Member", renames: list[Rename]): + self.members.append(member) + self.elements.update(member.elements) + self.renames += renames + + def described(self, mode: str) -> dict[LocationKey, tuple[str, str, str, str]]: + """:return: the kind, register name, name and description of every element.""" + described = {} + for member in self.members: + for register in member.shape.registers: + described.setdefault( + register_location(register, mode), ("R", register.name, register.name, register.description) + ) + for field in register.fields: + described.setdefault( + field_location(register, field, mode), ("F", register.name, field.name, field.description) + ) + return described + + def condense(self, mode: str = "binary", names: dict[str, str] = None): + """Splits the merged register map into the common core and the optional features.""" + presence = defaultdict(set) + instances = defaultdict(int) + for index, member in enumerate(self.members): + for location in member.elements.names: + presence[location].add(index) + instances[location] += len(member.instances) + total = len(self.members) + self.core = {location for location, shapes in presence.items() if len(shapes) == total} + features = defaultdict(set) + for location, shapes in presence.items(): + if len(shapes) != total: + features[frozenset(shapes)].add(location) + self.features = sorted( + ( + Feature(locations, shapes, max(instances[location] for location in locations)) + for shapes, locations in features.items() + ), + key=lambda f: (-f.instances, -len(f.locations)), + ) + described = self.described(mode) + for feature in self.features: + feature.elements = [described[ll] for ll in sorted(feature.locations) if ll in described] + feature.name = name_feature(feature.elements, names) + + +@dataclass +class Group: + """All instances of one peripheral type across all devices.""" + + name: str + instances: list[Instance] = _field(default_factory=list, repr=False) + variants: list[Variant] = _field(default_factory=list, repr=False) + mode: str = "binary" + ignored: list[str] = _field(default_factory=list) + """The names of the registers that were left out of the comparison.""" + + @property + def shapes(self) -> int: + return sum(len(variant.shapes) for variant in self.variants) + + @property + def devices(self) -> set[str]: + return {instance.device for instance in self.instances} + + +def _shapes(instances: list[Instance], ignored: re.Pattern = None) -> dict[Shape, list[Instance]]: + """Deduplicates the instances by their register map without the ignored registers.""" + shapes = defaultdict(list) + for instance in instances: + shapes[instance.shape.without(ignored) if ignored else instance.shape].append(instance) + # The largest register maps discriminate best and therefore seed the variants + return dict(sorted(shapes.items(), key=lambda s: (-len(s[0]), -len(s[1])))) + + +def variants_of( + name: str, instances: list[Instance], widening: bool = True, mode: str = "binary", names: dict[str, str] = None +) -> Group: + """ + Merges all instances of a peripheral group into as few variants as possible. + + Every shape is merged into the variant it shares the most elements with, so + that a shape that is compatible with several variants does not merely join + the first one. A shape that conflicts with every variant starts a new one. + + :param name: the name of the peripheral group, for example `TIM`. + :param instances: all instances of the group across all devices. + :param widening: whether a bit field that only grew into the reserved bits + above it is an optional extension instead of a conflict. + :param mode: `binary` locates elements by register address, `source` by + register name, and `similar` also tolerates differences in the + documentation, see `modm_data.svd2variants.merge.scope_of`. + :param names: the lookup table of generated feature names, see + `modm_data.svd2variants.annotate`. + :return: the group with its variants. + """ + group = Group(name, instances, mode=mode) + if ignored := IGNORED_REGISTERS.get(name): + group.ignored = sorted({r.name for i in instances for r in i.shape.registers if ignored.match(r.name)}) + for shape, shape_instances in _shapes(instances, ignored).items(): + member = Member(shape, ElementMap.from_shape(shape, mode), shape_instances) + best, shared, renames = None, -1, [] + for variant in group.variants: + difference = compare(variant.elements, member.elements, widening, mode == "similar") + if not difference.compatible: + continue + common = len(set(variant.elements.names) & set(member.elements.names)) + if common > shared: + best, shared, renames = variant, common, difference.renames + if best is None: + best = Variant(name) + group.variants.append(best) + renames = [] + best.add(member, renames) + + # A merged map collects aliases that are later evidence for a rename, so a + # conflict that kept two shapes apart may be gone once both variants are + # complete. Keep merging the smallest variant into its best match until no + # two variants are compatible anymore. + merged = True + while merged: + merged = False + for small in sorted(group.variants, key=lambda v: len(v.instances)): + best, shared, renames = None, -1, [] + for large in group.variants: + if large is small: + continue + difference = compare(large.elements, small.elements, widening, mode == "similar") + if not difference.compatible: + continue + common = len(set(large.elements.names) & set(small.elements.names)) + if common > shared: + best, shared, renames = large, common, difference.renames + if best is not None: + for member in small.members: + best.add(member, []) + best.renames += small.renames + renames + group.variants.remove(small) + merged = True + break + + for variant in group.variants: + variant.condense(mode, names) + group.variants.sort(key=lambda v: (-len(v.instances), -len(v.shapes))) + return group + + +def conflicts_between(variants: list[Variant], mode: str = "binary") -> dict[tuple[int, int], list[Conflict]]: + """:return: the conflicts between each pair of variants, which is why they are separate.""" + conflicts = {} + for left in range(len(variants)): + for right in range(left + 1, len(variants)): + difference = compare(variants[left].elements, variants[right].elements, similar=mode == "similar") + conflicts[(left, right)] = difference.conflicts + return conflicts diff --git a/src/modm_data/svd2variants/annotate.py b/src/modm_data/svd2variants/annotate.py new file mode 100644 index 0000000..f906d00 --- /dev/null +++ b/src/modm_data/svd2variants/annotate.py @@ -0,0 +1,316 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Naming Features with a Language Model + +The heuristic in `modm_data.svd2variants.naming` can only name a feature after +what its elements share lexically, which misses the semantics: the seventeen +bit fields of the first capture/compare channel of a timer share no name and no +description, but they are obviously "capture/compare channel 1". + +This module asks a small language model to name them instead and stores the +answers in a lookup table. The table is keyed by the *content* of the feature, +so it is stable across runs, reviewable as a diff, and only features that +actually changed need to be asked again. + +Names come from the first of these that works, so that a build never depends on +a model being there: + +1. the lookup table of a previous run, which the homepage and the CI download, +2. a local model served by LM Studio, see `local_endpoint`, or the `claude` + command line tool, +3. the heuristic of `modm_data.svd2variants.naming`. + +The prompt below is the result of measuring variants against each other on 91 +features of 12 peripheral groups. Keeping the numbers of similar elements as a +range and asking for a different name per feature both helped, while naming the +vague words to avoid produced more of them, and asking the model to first +explain what the elements have in common made the names worse. +""" + +import re +import json +import logging +import subprocess +import urllib.request +from hashlib import sha1 +from pathlib import Path +from multiprocessing.pool import ThreadPool + +from .naming import Element + +LOGGER = logging.getLogger(__name__) + +MODEL = "claude-haiku-4-5-20251001" +"""The model asked through the `claude` command line tool.""" +LOCAL_MODEL = "gemma-4-26b-a4b-it-qat-mlx" +"""The model asked through a local server, which LM Studio loads on demand.""" +LOCAL_ENDPOINT = "http://localhost:1234/v1" +"""Where LM Studio serves its OpenAI compatible API by default.""" +_MAX_ELEMENTS = 24 +_MAX_FEATURES = 16 + +_INSTRUCTIONS = """\ +You name the optional features of STM32 peripherals. Each block below lists registers and bit \ +fields of the STM32 {group} peripheral that exist on exactly the same set of devices, so together \ +they form one optional feature. + +Name each feature with a short noun phrase: +- 2 to 6 words in sentence case, without a trailing period. +- Say what the feature does, not which register it is in. Never use the word "register". +- Summarize many similar elements instead of listing them, but keep their numbers as a + range, e.g. "Lines 28 to 30 interrupt masking" or "Channels 3 and 4 DMA requests". +- Consider all elements, not only the first ones. +- Do not repeat the peripheral name {group}. +- Give every feature in the reply a different name, since they are different features. + +For example, these features of other peripherals: + +## a (5 elements in 4 registers) + CCMR1.CC1SEL — Capture/compare 1 selection + CCMR1.IC1PSC — Input capture 1 prescaler + CCR1.CCR1 — Capture/compare 1 value + DIER.CC1IE — Capture/compare 1 interrupt enable + SR.CC1IF — Capture/compare 1 interrupt flag + +## b (5 elements in 4 registers) + CR.WUTE — Wakeup timer enable + CR.WUTIE — Wakeup timer interrupt enable + WUTR.WUT — Wakeup auto-reload value + SR.WUTF — Wakeup timer flag + SCR.CWUTF — Clear wakeup timer flag + +## c (5 elements in 2 registers) + CR1.SMBHEN — SMBus host address enable + CR1.SMBDEN — SMBus device default address enable + CR1.ALERTEN — SMBus alert enable + CR1.PECEN — PEC enable + TIMEOUTR.TIMEOUTA — Bus timeout A + +## d (18 elements in 3 registers) + RXF0C.F0SA — Rx FIFO 0 start address + RXF0S.F0FL — Rx FIFO 0 fill level + IE.RF0NE — Rx FIFO 0 new message interrupt enable + RXF0C.F0S — Rx FIFO 0 size + ... and 14 more similar + +## e (2 elements in 2 registers) + POL.POL — Programmable polynomial + CR.POLYSIZE — Polynomial size + +are named: +{{"a": "Capture/compare channel 1", "b": "Wakeup timer", "c": "SMBus support", "d": "Receive FIFO 0", \ +"e": "Programmable polynomial"}} + +Reply with ONLY a JSON object mapping each id below to its name. +""" + + +def feature_key(elements: list[Element]) -> str: + """ + :param elements: the elements of one feature. + :return: a stable key for the content of the feature, so that a table entry + survives everything except a change to the feature itself. + """ + lines = sorted(f"{register}.{name}\t{description}" for _kind, register, name, description in elements) + return sha1("\n".join(lines).encode()).hexdigest()[:16] + + +def _sample(elements: list[Element]) -> list[Element]: + """ + Takes elements from all registers in turn, since small models tend to name a + feature after the first few elements, which are all in the same register. + """ + registers = {} + for element in elements: + registers.setdefault(element[1], []).append(element) + sample = [] + while len(sample) < min(_MAX_ELEMENTS, len(elements)): + for queue in registers.values(): + if queue and len(sample) < _MAX_ELEMENTS: + sample.append(queue.pop(0)) + return sample + + +def _block(index: int, elements: list[Element]) -> str: + registers = len({element[1] for element in elements}) + lines = [f"## {index} ({len(elements)} elements in {registers} registers)"] + for _kind, register, name, description in _sample(elements): + lines.append(f" {register}.{name}" + (f" — {description[:70]}" if description else "")) + if len(elements) > _MAX_ELEMENTS: + lines.append(f" ... and {len(elements) - _MAX_ELEMENTS} more similar") + return "\n".join(lines) + + +def _normalize(name: str) -> str | None: + """ + Brings a generated name into sentence case without a register suffix. + + :return: the name, or `None` if it is not a short noun phrase. + """ + name = " ".join(name.split()).strip(" .\"'") + name = re.sub(r"\s+registers?$", "", name, flags=re.IGNORECASE) + words = name.split(" ") + if not name or len(words) > 8 or len(name) > 70: + return None + + # Keep acronyms and mixed case words like IrDA, only lower title case words + def lower(word: str) -> str: + return "-".join(part.lower() if part[:1].isupper() and part[1:].islower() else part for part in word.split("-")) + + return " ".join([words[0][:1].upper() + words[0][1:]] + [lower(word) for word in words[1:]]) + + +def local_endpoint(endpoint: str = None, model: str = LOCAL_MODEL, timeout: float = 2) -> str | None: + """ + Looks for a local server that can name features, so that the names are only + generated where a model is actually available, see `modm_data.svd2variants`. + + :param endpoint: the URL to check, or the LM Studio default. + :param model: the model that the server must offer. + :return: the URL of the server, or `None` if it does not answer or lacks the model. + """ + url = endpoint or LOCAL_ENDPOINT + try: + with urllib.request.urlopen(url.rstrip("/") + "/models", timeout=timeout) as response: + models = {served.get("id") for served in json.loads(response.read()).get("data", [])} + except (OSError, ValueError): + return None + if model in models: + return url + LOGGER.info(f"{url} does not serve {model}, only {sorted(models)}") + return None + + +def _claude(prompt: str, model: str) -> str: + """Asks the model with the `claude` command line tool.""" + result = subprocess.run( + ["claude", "-p", "--model", model, "--allowed-tools", ""], + input=prompt, + capture_output=True, + text=True, + timeout=600, + ) + if result.returncode: + raise OSError(f"claude exited with {result.returncode}: {result.stderr[:200]}") + return result.stdout + + +def _openai(prompt: str, model: str, endpoint: str, reasoning: str = "none") -> str: + """ + Asks the model of an OpenAI compatible server, for example LM Studio at + `http://localhost:1234/v1`, Ollama or the llama.cpp server. + + :param reasoning: the reasoning effort. Reasoning models like Qwen3.5 think + for hundreds of tokens by default even for a short name, + and LM Studio only turns that off with `none`. + """ + request = urllib.request.Request( + endpoint.rstrip("/") + "/chat/completions", + data=json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": prompt}], + # Names should not change between runs + "temperature": 0, + "max_tokens": 2048, + **({"reasoning_effort": reasoning} if reasoning else {}), + } + ).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=900) as response: + answer = json.loads(response.read()) + return answer["choices"][0]["message"]["content"] or "" + + +def _ask( + group: str, batch: list[tuple[str, list[Element]]], model: str, endpoint: str = None, retry: bool = True +) -> dict[str, str]: + """Asks the model to name one batch of features of one peripheral group.""" + prompt = _INSTRUCTIONS.format(group=group) + "\n" + prompt += "\n\n".join(_block(index, elements) for index, (_key, elements) in enumerate(batch)) + try: + text = _openai(prompt, model, endpoint) if endpoint else _claude(prompt, model) + except (OSError, subprocess.TimeoutExpired, KeyError, IndexError, json.JSONDecodeError) as error: + LOGGER.warning(f"{group}: {error}") + return {} + + # Reasoning models may think out loud before answering + text = re.sub(r".*?", "", text, flags=re.DOTALL).strip() + if (start := text.find("{")) < 0 or (end := text.rfind("}")) < 0: + LOGGER.warning(f"{group}: no JSON in the answer: {text[:120]}") + return {} + try: + answers = json.loads(text[start : end + 1]) + except json.JSONDecodeError as error: + LOGGER.warning(f"{group}: {error}") + return {} + + names, missing = {}, [] + for index, (key, elements) in enumerate(batch): + answer = answers.get(str(index)) or answers.get(index) + if isinstance(answer, str) and (name := _normalize(answer)): + names[key] = name + else: + missing.append((key, elements)) + if not retry: + LOGGER.warning(f"{group}: no usable name for feature {index}: {answer!r}") + # Small models sometimes skip a few ids of a batch, which usually works on their own + if missing and retry: + names.update(_ask(group, missing, model, endpoint, retry=False)) + return names + + +def _batches(jobs: dict[str, list[tuple[str, list[Element]]]]) -> list[tuple[str, list]]: + batches = [] + for group, features in jobs.items(): + for start in range(0, len(features), _MAX_FEATURES): + batches.append((group, features[start : start + _MAX_FEATURES])) + return batches + + +def generate_names( + jobs: dict[str, list[tuple[str, list[Element]]]], + model: str = MODEL, + workers: int = 6, + progress=None, + endpoint: str = None, +) -> dict[str, str]: + """ + Asks the model to name every feature it is given. + + :param jobs: the features to name, as (key, elements) per peripheral group. + :param model: the model to ask. + :param workers: how many requests to run at the same time. + :param endpoint: the URL of an OpenAI compatible server instead of the + `claude` command line tool, e.g. `http://localhost:1234/v1`. + :param progress: called with the names of every finished batch, so that an + interrupted run, for example by a usage limit, keeps them. + :return: the generated names by feature key. + """ + batches = _batches(jobs) + names = {} + if not batches: + return names + with ThreadPool(workers) as pool: + results = pool.imap_unordered(lambda job: _ask(job[0], job[1], model, endpoint), batches) + for done, result in enumerate(results, 1): + names.update(result) + if progress is not None: + progress(result, done, len(batches)) + return names + + +def read_names(path: Path) -> dict[str, str]: + """:return: the lookup table of feature names, or an empty one.""" + if path is None or not Path(path).exists(): + return {} + return json.loads(Path(path).read_text()) + + +def write_names(names: dict[str, str], path: Path): + """Writes the lookup table sorted by key, so that it diffs cleanly.""" + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(json.dumps(dict(sorted(names.items())), indent=0, ensure_ascii=False) + "\n") diff --git a/src/modm_data/svd2variants/merge.py b/src/modm_data/svd2variants/merge.py new file mode 100644 index 0000000..59d16f2 --- /dev/null +++ b/src/modm_data/svd2variants/merge.py @@ -0,0 +1,527 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Comparing Register Maps + +Whether two register maps describe the same silicon cannot be decided by +comparing names alone, nor by comparing addresses alone: + +- ST renames registers and bit fields without changing the layout, most + visibly between the STM32H7x and STM32H7[RS]x devices. Comparing names alone + reports these as different implementations, which they are not. +- ST reuses layouts for entirely different registers. The STM32F1 `GPIO_CRL` + and the `GPIO_MODER` of every other device both start with two-bit fields at + the same positions, so comparing addresses alone merges the two GPIO + implementations into one, which they are not. + +Therefore both keys are compared, but they are *not* symmetric: + +- A name that appears at **two different locations** is a **conflict**: a + driver cannot be written against both maps, so they are different versions. + This is what separates the STM32F1 GPIO (`IDR` at `0x08`) from every other + GPIO (`IDR` at `0x10`). +- A location that carries **two different names** is a **rename**: the silicon + is the same and the names become aliases of each other. + +Names alone miss two more kinds of contradiction, since unrelated elements +never share a name: + +- Two bit fields with a different layout that **share bits** are a conflict, + unless they are alternates of each other in one map, see `overlaps`. This is + what separates the STM32L1 `COMP_CSR_OUTSEL[23:21]` from the STM32G0 + `COMP_CSR_BLANKING[24:20]`, which would otherwise each become an optional + feature of the other. +- A rename is only believable **inside the same register**, see `reuses`. The + STM32F373 `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` share a + bit, but the registers share neither a name nor a single bit field, so the + bit was reused, not renamed. + +Everything else is a *population* difference: an element that only exists in +one of the maps is an optional feature of the merged map. + +## Binary and Source Compatibility + +Many register maps differ only in *where* a register sits, while the register +itself is unchanged. Which of the two matters depends on how the hardware is +addressed, so the comparison runs in two modes, see `scope_of`: + +- `binary`: elements are located by the address of their register, so moving a + register to another offset is a conflict. This is what a driver that pokes + at fixed addresses needs. +- `source`: elements are located by the *name* of their register, so a + register may move as long as it keeps its name and its layout. This is what + code written against a CMSIS header needs, since the structure member + abstracts the offset away. A renamed register is a different register here, + and simply becomes an optional feature of the merged map. +- `similar`: like `source`, but conflicts that only exist in the + documentation are tolerated, see `documentation_only`. +""" + +import re +from dataclasses import dataclass, field as _field +from collections import defaultdict +from .model import Shape, LocationKey, NameKey + + +@dataclass +class Conflict: + """A name that is located differently in two register maps.""" + + name: NameKey + locations: set[LocationKey] + other: set[LocationKey] + + @property + def kind(self) -> str: + """ + - `widened`: a bit field that starts at the same position but is longer + in one of the maps, since ST tends to grow a field into the bits that + were reserved before, for example the `IWDG_PR` prescaler. + - `resized`: an element with the same name and address, but a different + size, which is not merely an extension of the shorter one. + - `moved`: an element with the same name at a different address. + """ + locations = self.locations | self.other + if self.name[0] == "F": + # Same register and same start bit, only the length of the field differs + return "widened" if len({location[1:3] for location in locations}) == 1 else "moved" + return "resized" if len({location[1] for location in locations}) == 1 else "moved" + + def __str__(self) -> str: + def location(keys): + return "/".join(sorted(_format_location(key) for key in keys)) + + return f"{_format_name(self.name)}: {self.kind} {location(self.locations)} != {location(self.other)}" + + +_INDEX = re.compile(r"_?\d+$") + + +@dataclass +class Overlap: + """ + Two bit fields with a different layout that share bits, without being + alternates of each other in any single register map. + """ + + location: LocationKey + names: set[NameKey] + other: LocationKey + others: set[NameKey] + + @property + def kind(self) -> str: + return "overlap" + + def __str__(self) -> str: + def label(location, names): + return f"{'/'.join(sorted(name[-1] for name in names))}[{_format_range(location)}]" + + return ( + f"{_format_scope(self.location[1])}: {label(self.location, self.names)} " + f"overlaps {label(self.other, self.others)}" + ) + + +def _format_range(location: LocationKey) -> str: + position, width = location[2], location[3] + return f"{position}" if width == 1 else f"{position + width - 1}:{position}" + + +def _same_field(names: set[NameKey], others: set[NameKey]) -> bool: + """ + Whether two names describe the same bit field, which either the name rules + already judge, or which one header describes bit by bit, e.g. `PLLM` and `PLLM_2`. + """ + for name in names: + for other in others: + if name[-1] == other[-1] or _INDEX.sub("", name[-1]) == other[-1] or _INDEX.sub("", other[-1]) == name[-1]: + return True + return False + + +def overlaps(left: "ElementMap", right: "ElementMap") -> list[Overlap]: + """ + Unrelated bit fields cannot share bits, but their names never collide, so + the name rules cannot see it. The STM32L1 `COMP_CSR_OUTSEL[23:21]` and the + STM32G0 `COMP_CSR_BLANKING[24:20]` are both at the same offset in the same + register, and without this check they merge into one register map, where + each becomes an optional feature of the other. + + :return: the new fields of `right` that overlap a differently laid out + field of `left`, which `right` does not have as an alternate. + """ + fields = defaultdict(list) + for location in left.names: + if location[0] == "F": + fields[location[1]].append(location) + found = [] + for location, names in right.names.items(): + if location[0] != "F" or location in left.names: + continue + _kind, scope, position, width = location + for other in fields.get(scope, ()): + if other[2] >= position + width or position >= other[2] + other[3]: + continue + # Alternates of the same register in one map may overlap, e.g. TIM_CCMR1 input and output + if other in right.names or _same_field(names, left.names[other]): + continue + found.append(Overlap(location, set(names), other, set(left.names[other]))) + return found + + +@dataclass +class Reuse: + """ + A location that is named differently in two register maps, but inside two + unrelated registers, so the names only coincide in the bits they use. + """ + + location: LocationKey + names: set[NameKey] + others: set[NameKey] + registers: set[str] + other_registers: set[str] + + @property + def kind(self) -> str: + return "reused" + + def __str__(self) -> str: + def label(names, registers): + return f"{'/'.join(sorted(registers))}.{'/'.join(sorted(name[-1] for name in names))}" + + where = _format_scope(self.location[1]) + if self.location[0] == "F": + where += f"[{_format_range(self.location)}]" + return f"{where}: {label(self.names, self.registers)} reused as {label(self.others, self.other_registers)}" + + +def _register_names(elements: "ElementMap") -> dict: + names = defaultdict(set) + for location, keys in elements.names.items(): + if location[0] == "R": + names[location[1]] |= {key[-1] for key in keys} + return names + + +def reuses(left: "ElementMap", right: "ElementMap", renames: list["Rename"]) -> tuple[list[Reuse], list["Rename"]]: + """ + A rename is only believable inside the same register. The STM32F373 + `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` sit at the same + bit, but the registers share neither a name nor a single bit field, so the + bit was reused for something else rather than renamed. Registers count as + the same if they share a name, or if at least one bit field has the same + name at the same location, e.g. `GPIO_AFRL` and `GPIO_AFR[0]`. + + :return: the renames that are actually reuses, and the remaining renames. + """ + left_registers, right_registers = _register_names(left), _register_names(right) + shared = { + location[1] + for location, names in right.names.items() + if location[0] == "F" and location in left.names and not left.names[location].isdisjoint(names) + } + # A register without bit fields on either side gives no evidence, e.g. an array whose bits were not named + left_fields = {location[1] for location in left.names if location[0] == "F"} + fields = left_fields & {location[1] for location in right.names if location[0] == "F"} + + reused, kept = [], [] + for rename in renames: + scope = rename.location[1] + same = not left_registers[scope].isdisjoint(right_registers[scope]) or scope in shared + if same or scope not in fields: + kept.append(rename) + else: + reused.append( + Reuse(rename.location, rename.names, rename.other, left_registers[scope], right_registers[scope]) + ) + return reused, kept + + +@dataclass +class Rename: + """A location that is named differently in two register maps.""" + + location: LocationKey + names: set[NameKey] + other: set[NameKey] + + def __str__(self) -> str: + def name(keys): + return "/".join(sorted(_format_name(key) for key in keys)) + + return f"{_format_location(self.location)}: {name(self.names)} = {name(self.other)}" + + +MODES = ("binary", "source", "similar") +"""The compatibility modes, see `scope_of` and `documentation_only`.""" + + +def scope_of(register, mode: str): + """ + The scope that an element is located in, which is what the two modes differ in: + + - `binary`: the offset of the register, so two maps are only compatible if + every register is at the same address. This is what a driver that pokes + at fixed addresses needs. + - `source`: the name of the register, so the offsets are free to move as + long as the register keeps its name and its layout. This is what code + written against a CMSIS header needs, since the header abstracts the + offsets away into structure members. + - `similar`: like `source`, but differences that only exist in the + documentation are no conflicts, see `documentation_only`. + """ + return register.offset if mode == "binary" else register.name + + +def register_location(register, mode: str) -> LocationKey: + return ("R", scope_of(register, mode), register.width, register.dim) + + +def register_name(register) -> NameKey: + return ("R", register.name) + + +def field_location(register, field, mode: str) -> LocationKey: + return ("F", scope_of(register, mode), field.position, field.width) + + +def field_name(register, field, mode: str) -> NameKey: + return ("F", scope_of(register, mode), field.name) + + +def _format_scope(scope) -> str: + return f"0x{scope:03x}" if isinstance(scope, int) else str(scope) + + +def _format_location(key: LocationKey) -> str: + if key[0] == "R": + return _format_scope(key[1]) + position, width = key[2], key[3] + return f"{_format_scope(key[1])}[{position if width == 1 else f'{position + width - 1}:{position}'}]" + + +def _format_name(key: NameKey) -> str: + return key[1] if key[0] == "R" else f"{_format_scope(key[1])}.{key[2]}" + + +class ElementMap: + """ + A bidirectional map between the locations and the names of the elements of + one or more register maps. Merged maps keep every alias of a location and + are therefore able to detect a conflict with any of them. + """ + + def __init__(self): + self.names: dict[LocationKey, set[NameKey]] = defaultdict(set) + self.locations: dict[NameKey, set[LocationKey]] = defaultdict(set) + + @classmethod + def from_shape(cls, shape: Shape, mode: str = "binary") -> "ElementMap": + elements = cls() + for register in shape.registers: + elements.add(register_location(register, mode), register_name(register)) + for field in register.fields: + elements.add(field_location(register, field, mode), field_name(register, field, mode)) + return elements + + def add(self, location: LocationKey, name: NameKey): + self.names[location].add(name) + self.locations[name].add(location) + + def update(self, other: "ElementMap"): + for location, names in other.names.items(): + self.names[location] |= names + for name, locations in other.locations.items(): + self.locations[name] |= locations + + @property + def scopes(self) -> set: + """The offsets or the names of all registers, depending on the mode.""" + return {location[1] for location in self.names if location[0] == "R"} + + def compare(self, other: "ElementMap") -> tuple[list[Conflict], list[Rename]]: + """ + :return: the names that are located differently in `other` and the + locations that are named differently in `other`. + """ + conflicts = [ + Conflict(name, set(self.locations[name]), set(locations)) + for name, locations in other.locations.items() + if name in self.locations and self.locations[name].isdisjoint(locations) + ] + renames = [ + Rename(location, set(self.names[location]), set(names)) + for location, names in other.names.items() + if location in self.names and self.names[location].isdisjoint(names) + ] + return conflicts, renames + + +@dataclass +class Difference: + """The result of comparing two register maps.""" + + conflicts: list[Conflict | Overlap | Reuse] = _field(default_factory=list) + widened: list[Conflict] = _field(default_factory=list) + tolerated: list = _field(default_factory=list) + """Conflicts that only differ in the documentation, see `documentation_only`.""" + renames: list[Rename] = _field(default_factory=list) + only_left: set[LocationKey] = _field(default_factory=set) + only_right: set[LocationKey] = _field(default_factory=set) + disjoint: bool = False + + @property + def compatible(self) -> bool: + """Whether both register maps can be merged into one.""" + return not self.conflicts and not self.disjoint + + @property + def identical(self) -> bool: + """Whether both register maps describe the same elements at the same locations.""" + return self.compatible and not self.only_left and not self.only_right and not self.renames + + @property + def relation(self) -> str: + if not self.compatible: + return "incompatible" + if not self.only_left and not self.only_right and not self.widened: + return "renamed" if self.renames else "identical" + if not self.only_right: + return "superset" + if not self.only_left: + return "subset" + return "overlapping" + + +def _bits(location: LocationKey) -> set[int]: + return set(range(location[2], location[2] + location[3])) + + +def _plain(name: str) -> str: + """The name without indices, placeholders and separators, e.g. `COMPBLANKING` for `COMPxBLANKING`.""" + return re.sub(r"[\d_]|x(?=[A-Z])", "", name).upper() + + +def _related(names: set[NameKey], others: set[NameKey], width: int) -> bool: + """ + Whether two bit field names are the same without their indices, e.g. `EM_12` + and `EM47`, one contains the other, e.g. `BLANKING` and `COMPxBLANKING`, or + they start the same, e.g. `FPU_IE` and `FPU_IOIE`. + + Different indices usually mean different instances of a bit field, e.g. the + `LPGPIO_MODER_MOD1[1]` of pin 1 inside the `GPIO_MODER_MODE0[1:0]` of pin 0, + unless the wider bit field is a mask of many, e.g. `EXTI_EMR2_EM[31:15]`. + + :param width: the width of the wider bit field. + """ + for name in names: + for other in others: + indices = [re.findall(r"\d+", key[-1]) for key in (name, other)] + if all(indices) and indices[0] != indices[1] and width < 4: + continue + short, long = sorted((_plain(name[-1]), _plain(other[-1])), key=len) + if short and (short == long or (len(short) >= 4 and short in long) or long.startswith(short[:3])): + return True + return False + + +def documentation_only(conflicts: list) -> tuple[list, list]: + """ + Separates the conflicts that are only differences in how a register is + documented from the ones that are differences in the silicon: + + - A register declared with another width, e.g. `USART_RDR` as `uint16_t` + in the STM32F0 header and as `uint32_t` everywhere else. + - A bit field with the same name over overlapping bits, e.g. a mask of many + bit fields that covers a different range. + - A bit field described as smaller bit fields with a related name, e.g. + `COMP_CSR_BLANKING` and `COMP_CSR_COMPxBLANKING`. + - A bit field of at least four bits described as smaller bit fields that + tile one of its edges, if they are a value split into at least two parts + of at least four bits, e.g. `USART_BRR_BRR` as `USART_BRR_DIV_MANTISSA` + and `USART_BRR_DIV_FRACTION`, or flags at the edge of a value that take + up at most a quarter of it, e.g. the `TIM_CNT_UIFCPY` flag in the top bit + of `TIM_CNT_CNT`. + + Unrelated bit fields nest by coincidence surprisingly often, which is not + tolerated: small ones, e.g. `DBGMCU_CR_TRACE_MODE[7:6]` and + `DBGMCU_CR_DBG_STOPD3[7]`, flags in another position, e.g. the STM32F3 + `SYSCFG_CFGR1_FPU_IE[31:26]` and the STM32F0 `SYSCFG_CFGR1_USART3_DMA_RMP[26]`, + and single wide bit fields, e.g. the STM32F4 `FLASH_OPTCR1_nWRP[27:16]` and + the STM32F7 `FLASH_OPTCR1_BOOT_ADD1[31:16]`. + + :return: the conflicts that differ in the silicon and the ones that don't. + """ + # All bit fields that nest inside a wider bit field, grouped by the wider one + nested, pairs = defaultdict(list), {} + for conflict in conflicts: + if isinstance(conflict, Overlap): + this, that = (conflict.location, conflict.names), (conflict.other, conflict.others) + inner, outer = sorted((this, that), key=lambda field: field[0][3]) + if _bits(inner[0]) <= _bits(outer[0]): + nested[outer[0]].append(inner[0]) + pairs[id(conflict)] = (inner, outer) + + def decomposes(outer: LocationKey) -> bool: + inners = nested[outer] + bits = set().union(*(_bits(inner) for inner in inners)) + low, high = min(bits), max(bits) + edge = bits == set(range(low, high + 1)) and (low == outer[2] or high == outer[2] + outer[3] - 1) + if outer[3] < 4 or not edge: + return False + parts = len(inners) >= 2 and all(inner[3] >= 4 for inner in inners) + flags = len(bits) * 4 <= outer[3] + return parts or flags + + silicon, documentation = [], [] + for conflict in conflicts: + if isinstance(conflict, Conflict): + locations = conflict.locations | conflict.other + same_bits = conflict.name[0] == "F" and bool(set.intersection(*(_bits(ll) for ll in locations))) + (documentation if conflict.kind == "resized" or same_bits else silicon).append(conflict) + elif (pair := pairs.get(id(conflict))) is not None: + inner, outer = pair + related = _related(inner[1], outer[1], outer[0][3]) + (documentation if related or decomposes(outer[0]) else silicon).append(conflict) + else: + silicon.append(conflict) + return silicon, documentation + + +def compare(left: ElementMap, right: ElementMap, widening: bool = True, similar: bool = False) -> Difference: + """ + Compares two register maps, see the module documentation for the rules. + + :param left: the register map to compare against, may be a merged map. + :param right: the register map to compare. + :param widening: whether a bit field that only grew into the reserved bits + above it is an optional extension instead of a conflict. + :param similar: whether differences in the documentation are tolerated, see + `documentation_only`. + :return: the differences between both maps. + """ + conflicts, renames = left.compare(right) + conflicts += overlaps(left, right) + reused, renames = reuses(left, right, renames) + conflicts += reused + widened = [] + if widening: + widened = [conflict for conflict in conflicts if conflict.kind == "widened"] + conflicts = [conflict for conflict in conflicts if conflict.kind != "widened"] + tolerated = [] + if similar: + conflicts, tolerated = documentation_only(conflicts) + # Two maps without a single common register describe unrelated hardware + disjoint = bool(left.names) and bool(right.names) and left.scopes.isdisjoint(right.scopes) + return Difference( + conflicts=conflicts, + widened=widened, + tolerated=tolerated, + renames=renames, + only_left=set(left.names) - set(right.names), + only_right=set(right.names) - set(left.names), + disjoint=disjoint, + ) diff --git a/src/modm_data/svd2variants/model.py b/src/modm_data/svd2variants/model.py new file mode 100644 index 0000000..edf2134 --- /dev/null +++ b/src/modm_data/svd2variants/model.py @@ -0,0 +1,143 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +The canonical description of a peripheral register map and its atomic elements. + +A register map is compared as a set of *elements*, where every element has both +a *location* and a *name*: + +- a register: `("R", offset, width, dim)` located, `("R", name)` named. +- a bit field inside that register: `("F", offset, position, width)` located, + `("F", offset, name)` named, so that the field names of a renamed register + still line up. + +Comparing both keys separately is what makes the difference between a register +map that was merely renamed and one that was actually changed, see +`modm_data.svd2variants.merge`. +""" + +import re +from dataclasses import dataclass, field as _field + +# Locations and names are compared as plain tuples for speed +LocationKey = tuple +NameKey = tuple + + +@dataclass(frozen=True, slots=True, order=True) +class Field: + """A bit field of a register.""" + + position: int + width: int + name: str + # Descriptions are documentation, not layout, and must not split a variant + description: str = _field(default="", compare=False) + + @property + def mask(self) -> int: + return ((1 << self.width) - 1) << self.position + + @property + def bit_range(self) -> str: + return f"{self.position}" if self.width == 1 else f"{self.position + self.width - 1}:{self.position}" + + +@dataclass(frozen=True, slots=True, order=True) +class Register: + """A register with its bit fields.""" + + offset: int + name: str + width: int = 4 + dim: int = 0 + fields: tuple[Field, ...] = () + description: str = _field(default="", compare=False) + # The CMSIS headers declare almost every register `__IO`, so the access is too + # sparse to tell implementations apart and is only carried along for display + access: str = _field(default="", compare=False) + + +@dataclass(frozen=True, slots=True) +class Shape: + """ + The register map of one peripheral instance, normalized so that two + instances with the same memory layout compare and hash equal. + + Registers sharing an offset are alternates of each other (unions in the + CMSIS header, for example `TIM_CCMR1` in output and input mode). They are + kept as separate registers, but their fields share one location scope, + since they describe the same physical register. + """ + + registers: tuple[Register, ...] + + @staticmethod + def from_peripheral(peripheral) -> "Shape": + """:param peripheral: a `modm_data.svd.Peripheral` tree node.""" + registers = [] + for register in peripheral.children: + fields = tuple( + sorted( + Field(f.position, f.width, f.name, getattr(f, "description", "") or "") for f in register.children + ) + ) + registers.append( + Register( + register.offset, + register.name, + register.width, + getattr(register, "dim", 0) or 0, + fields, + getattr(register, "description", "") or "", + getattr(register, "access", "") or "", + ) + ) + return Shape(tuple(sorted(registers))) + + def without(self, pattern: re.Pattern) -> "Shape": + """:return: the register map without the registers whose name matches the pattern.""" + return Shape(tuple(register for register in self.registers if not pattern.match(register.name))) + + @property + def offsets(self) -> set[int]: + return {register.offset for register in self.registers} + + def __len__(self) -> int: + return len(self.registers) + sum(len(register.fields) for register in self.registers) + + def __hash__(self) -> int: + return hash(self.registers) + + +IGNORED_REGISTERS: dict[str, re.Pattern] = { + # The option registers route device signals to the timer inputs, e.g. which + # comparator drives the break input or which pin is remapped to TI1. Their + # layout follows the device, not the timer, so they would split every timer + # implementation into one variant per device family. + "TIM": re.compile(r"^(OR\d?|AF\d|TISEL)$"), +} +"""Registers per peripheral group that are not part of the silicon implementation.""" + + +@dataclass +class Instance: + """One peripheral of one device.""" + + device: str + name: str + group: str + address: int + shape: Shape = _field(repr=False) + + +def group_name(peripheral) -> str: + """ + :param peripheral: a `modm_data.svd.Peripheral` tree node. + :return: the CMSIS structure type of the peripheral, for example `TIM` for + `TIM_TypeDef`, or the instance name without its index as fallback. + """ + if group := getattr(peripheral, "group", None): + return group + return re.sub(r"\d+", "", peripheral.name) or peripheral.name diff --git a/src/modm_data/svd2variants/naming.py b/src/modm_data/svd2variants/naming.py new file mode 100644 index 0000000..5155ee6 --- /dev/null +++ b/src/modm_data/svd2variants/naming.py @@ -0,0 +1,108 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Naming a Feature + +A feature is a set of elements that appear in exactly the same instances, which +is a mechanical definition and says nothing about what the feature *is*. Most +of them do have something in common though, since ST names and documents +related elements alike, so the name is derived from whatever the elements share: + +- a single element simply uses its own name and description, +- `RCC.DACEN`, `RCC.DACRST` and `RCC.DACLPEN` share the name prefix `DAC`, +- `EXTI.RT27` and `EXTI.FT27` share the description "trigger ... on line 27", +- `DMA.CCR4` … `DMA.CCR6` share the register name prefix `CCR`. + +About four out of five features are named this way. The rest are genuinely +heterogeneous, mostly large `RCC` and `DMA` features that enable a whole set of +unrelated peripherals or channels at once, and fall back to a plain count. +""" + +import re + +Element = tuple[str, str, str, str] +"""kind (`R` or `F`), register name, element name, description.""" + +_WORDS = re.compile(r"[A-Za-z0-9/]+") +# Words that carry no meaning on their own and make for a poor feature name +_FILLER = {"the", "and", "for", "bit", "bits", "register", "registers", "this", "not", "used"} + + +def _common_prefix(names: list[str]) -> str: + head, tail = min(names), max(names) + length = 0 + while length < len(head) and length < len(tail) and head[length] == tail[length]: + length += 1 + return head[:length].rstrip("_") + + +def _common_phrase(descriptions: list[str]) -> str: + """:return: the longest run of words that appears in every description.""" + words = [_WORDS.findall(description) for description in descriptions] + if not all(words): + return "" + shortest = min(words, key=len) + best = [] + for start in range(len(shortest)): + for end in range(len(shortest), start + len(best), -1): + phrase = [word.lower() for word in shortest[start:end]] + if all( + any( + phrase == [w.lower() for w in other[i : i + len(phrase)]] + for i in range(len(other) - len(phrase) + 1) + ) + for other in words + ): + best = shortest[start:end] + break + return " ".join(best).strip(" ,.:;-") + + +def name_feature(elements: list[Element], names: dict[str, str] = None) -> str: + """ + :param elements: the elements of one feature. + :param names: the lookup table of generated names, see + `modm_data.svd2variants.annotate`, which takes precedence. + :return: a short human readable name, see the module documentation. + """ + if not elements: + return "" + if names: + from .annotate import feature_key + + if generated := names.get(feature_key(elements)): + return generated + names = [element[2] for element in elements] + registers = sorted({element[1] for element in elements}) + descriptions = [element[3] for element in elements if element[3]] + + if len(elements) == 1: + kind, register, name, description = elements[0] + label = name if kind == "R" else f"{register}.{name}" + return f"{label} — {description}" if description else label + + if len(set(names)) == 1: + return f"{names[0]} in {len(registers)} registers" + + phrase = _common_phrase(descriptions) if len(descriptions) == len(elements) else "" + if len(phrase.split()) >= 2: + return phrase + + if len(prefix := _common_prefix(names)) >= 3: + return f"{prefix}*" + + if len(registers) == 1: + return f"{registers[0]}: {len(elements)} bit fields" + + if len(registers) > 1 and len(prefix := _common_prefix(registers)) >= 3: + return f"{prefix}* in {len(registers)} registers" + + if len(registers) <= 3: + return ", ".join(registers) + + # A single shared word is a weak name, but still better than a bare count + if len(phrase) >= 4 and phrase.lower() not in _FILLER: + return f"{phrase} in {len(registers)} registers" + + return f"{len(elements)} elements in {len(registers)} registers" diff --git a/src/modm_data/svd2variants/read.py b/src/modm_data/svd2variants/read.py new file mode 100644 index 0000000..ebe3151 --- /dev/null +++ b/src/modm_data/svd2variants/read.py @@ -0,0 +1,36 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +"""Reads the peripheral instances of many SVD files into one flat list.""" + +from pathlib import Path +from collections import defaultdict +from multiprocessing.pool import ThreadPool + +from ..svd import read_svd +from .model import Instance, Shape, group_name + + +def instances_of_svd(path: Path) -> list[Instance]: + """:return: every peripheral instance of one SVD file.""" + device = read_svd(path) + return [ + Instance( + device.name, peripheral.name, group_name(peripheral), peripheral.address, Shape.from_peripheral(peripheral) + ) + for peripheral in device.children + ] + + +def instances_of_svds(paths: list[Path]) -> dict[str, list[Instance]]: + """ + :param paths: the SVD files to read. + :return: every peripheral instance of every SVD file grouped by peripheral type. + """ + with ThreadPool() as pool: + results = pool.map(instances_of_svd, paths) + groups = defaultdict(list) + for instances in results: + for instance in instances: + groups[instance.group].append(instance) + return dict(sorted(groups.items())) diff --git a/src/modm_data/svd2variants/write.py b/src/modm_data/svd2variants/write.py new file mode 100644 index 0000000..ee377b3 --- /dev/null +++ b/src/modm_data/svd2variants/write.py @@ -0,0 +1,171 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +Serializes the peripheral variants into the JSON files that the Peripheral +Variant Explorer loads: one index of all groups and one file per group. + +Device names are shared per file and referenced by index, since the same few +hundred names would otherwise be repeated for every single variant, feature +and register map. +""" + +import json +from pathlib import Path +from collections import defaultdict +from .analyze import Group, Variant, conflicts_between +from .merge import ElementMap, register_location, field_location + + +def _names(elements: ElementMap, location: tuple) -> list[str]: + """:return: every name the element is known under, the most common one first.""" + names = elements.names.get(location, set()) + return sorted(name[-1] for name in names) + + +def _register_map(variant: Variant, features: dict[tuple, int], mode: str) -> list[dict]: + """The merged register map of the variant, with every element tagged by its feature.""" + registers = {} + descriptions = {} + offsets = defaultdict(set) + accesses = defaultdict(set) + for member in variant.members: + for register in member.shape.registers: + location = register_location(register, mode) + entry = registers.setdefault( + location, + { + "width": register.width, + "dim": register.dim, + "names": _names(variant.elements, location), + "feature": features.get(location, -1), + "fields": {}, + }, + ) + # In source mode the same register may sit at a different offset per instance + offsets[location].add(register.offset) + descriptions.setdefault(location, register.description) + if register.access: + accesses[location].add(register.access) + for field in register.fields: + flocation = field_location(register, field, mode) + entry["fields"].setdefault( + flocation, + { + "position": field.position, + "width": field.width, + "names": _names(variant.elements, flocation), + "feature": features.get(flocation, -1), + "description": field.description, + }, + ) + for location, entry in registers.items(): + entry["description"] = descriptions.get(location, "") + entry["offset"] = min(offsets[location]) + # Instances may disagree, e.g. a register that only some headers declare read-only + entry["access"] = sorted(accesses[location]) + if len(offsets[location]) > 1: + entry["offsets"] = sorted(offsets[location]) + entry["fields"] = sorted(entry.pop("fields").values(), key=lambda f: f["position"]) + return sorted(registers.values(), key=lambda r: (r["offset"], r["names"])) + + +def _variant(variant: Variant, index: int, devices: dict[str, int], conflicts: dict, mode: str) -> dict: + features = {location: index for index, feature in enumerate(variant.features) for location in feature.locations} + return { + "instances": len(variant.instances), + "devices": sorted(devices[device] for device in variant.devices), + "names": sorted(variant.names), + "members": [ + { + "instances": len(member.instances), + "devices": sorted(devices[device] for device in member.devices), + "names": sorted({instance.name for instance in member.instances}), + } + for member in variant.members + ], + "features": [ + { + "name": feature.name, + "elements": len(feature.locations), + "registers": len(feature.registers), + "instances": feature.instances, + "members": sorted(feature.shapes), + "devices": sorted( + {devices[device] for member in feature.shapes for device in variant.members[member].devices} + ), + } + for feature in variant.features + ], + "renames": sorted({str(rename) for rename in variant.renames}), + "conflicts": { + str(other): [str(conflict) for conflict in found] + for (left, other), found in conflicts.items() + if left == index and found + }, + "registers": _register_map(variant, features, mode), + } + + +def group_to_json(group: Group) -> dict: + """:return: the group with all of its variants as a JSON compatible dictionary.""" + devices = {device: index for index, device in enumerate(sorted(group.devices))} + conflicts = conflicts_between(group.variants, group.mode) + return { + "name": group.name, + "mode": group.mode, + "ignored": group.ignored, + "instances": len(group.instances), + "devices": sorted(devices), + "variants": [ + _variant(variant, index, devices, conflicts, group.mode) for index, variant in enumerate(group.variants) + ], + } + + +def index_to_json(groups: list[Group]) -> dict: + """:return: the summary of all groups as a JSON compatible dictionary.""" + return { + "groups": [ + { + "name": group.name, + "ignored": group.ignored, + "instances": len(group.instances), + "devices": len(group.devices), + "shapes": group.shapes, + "variants": [ + { + "instances": len(variant.instances), + "devices": len(variant.devices), + "members": len(variant.members), + "registers": len({location for location in variant.elements.names if location[0] == "R"}), + "core": len(variant.core), + "features": len(variant.features), + } + for variant in group.variants + ], + } + for group in groups + ], + "instances": sum(len(group.instances) for group in groups), + "shapes": sum(group.shapes for group in groups), + "variants": sum(len(group.variants) for group in groups), + } + + +def write_variants(modes: dict[str, list[Group]], path: Path): + """ + Writes `variants.json` with the summary of every group in every mode and + `/.json` with the full register maps of every group. + + :param modes: the analyzed peripheral groups per compatibility mode. + :param path: the output folder. + """ + index = {} + for mode, groups in modes.items(): + folder = path / mode + folder.mkdir(parents=True, exist_ok=True) + for group in groups: + (folder / f"{group.name}.json").write_text(json.dumps(group_to_json(group), separators=(",", ":")) + "\n") + index[mode] = index_to_json(groups) + (path / "variants.json").write_text(json.dumps({"modes": index}, indent=0) + "\n") From 0d6212d5a063767c1cfd730e804526be0749fcce Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Thu, 17 Sep 2026 21:39:59 +0200 Subject: [PATCH 4/5] [docs] Publish the peripheral variants with the Variant Explorer Generates the variant JSON files into the documentation folder and adds the Peripheral Variant Explorer to the homepage, which is fetched from the same gist as the SVD Explorer together with the table of generated feature names, so that neither the page nor the names need to be committed here. The variants are also generated in CI and attached to releases as variants-archive.zip. --- .github/workflows/test-svd.yml | 26 ++++- CHANGELOG.md | 7 ++ README.md | 2 + docs/.gitignore | 1 + docs/mkdocs.yml | 2 + docs/src/pipeline/svd2variants.md | 163 ++++++++++++++++++++++++++++++ tools/make/common.mk | 6 ++ 7 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 docs/src/pipeline/svd2variants.md diff --git a/.github/workflows/test-svd.yml b/.github/workflows/test-svd.yml index b12cf6a..f8ad001 100644 --- a/.github/workflows/test-svd.yml +++ b/.github/workflows/test-svd.yml @@ -54,10 +54,33 @@ jobs: path: svd-archive/ retention-days: 7 - - name: Zip SVD Files + - name: Analyze Peripheral Variants + run: | + curl -fsSL --create-dirs -o ext/cache/svd2variants/feature-names-gemma.json \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/feature-names-gemma.json || true + python3 -m modm_data.svd2variants --all --input svd-archive/ --names ext/cache/svd2variants/feature-names-gemma.json \ + --no-generate --output variants-archive/ + + - name: Upload Variant Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: variant-logs + path: log/stmicro/variants + retention-days: 1 + + - name: Upload Peripheral Variants + uses: actions/upload-artifact@v4 + with: + name: variants-archive + path: variants-archive/ + retention-days: 7 + + - name: Zip SVD Files and Peripheral Variants if: ${{ github.event_name == 'release' && github.event.action == 'published' }} run: | zip -r svd-archive.zip svd-archive/ + zip -r variants-archive.zip variants-archive/ - name: Upload to Github Release uses: softprops/action-gh-release@v2 @@ -65,3 +88,4 @@ jobs: with: files: | svd-archive.zip + variants-archive.zip diff --git a/CHANGELOG.md b/CHANGELOG.md index 72cc27a..a7acc32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- Merge the register maps of all STM32 peripheral instances into the smallest + set of binary, source and similar compatible variants and publish them with + the Peripheral Variant Explorer. +- Add the register access to the SVD files reconstructed from the CMSIS headers. + ## 0.0.4 - Move the data extraction pipelines of all vendors from modm-devices. diff --git a/README.md b/README.md index ff279e4..4b261af 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,9 @@ flowchart LR B(HTML) -->|html| C C(Python) -->|owl| E D(CMSIS-SVD) -->|cmsis-svd| C + D -->|svd2variants| K E[OWL] + K[Peripheral Variants] F(CMSIS Header) -->|header2svd| D I(Open Pin Data) -->|cubemx| C G(CubeMX) -->|cubemx| C diff --git a/docs/.gitignore b/docs/.gitignore index f2e9bc7..c81190d 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -2,6 +2,7 @@ src/api/ src/pinout/ src/svd/ +src/variants/ src/index.md src/source/overview.md src/pipeline/overview.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ff1e072..68a7c45 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -87,6 +87,8 @@ nav: - HTML to SVD: pipeline/html2svd.md #- HTML to OWL: pipeline/html2owl.md - Header to SVD: pipeline/header2svd.md + - SVD to Variants: pipeline/svd2variants.md - API Reference: api/modm_data.html - Pinouts: pinout/index.html - SVD Explorer: svd/index.html + - Peripheral Variants: variants/index.html diff --git a/docs/src/pipeline/svd2variants.md b/docs/src/pipeline/svd2variants.md new file mode 100644 index 0000000..9d9506e --- /dev/null +++ b/docs/src/pipeline/svd2variants.md @@ -0,0 +1,163 @@ +# SVD to Peripheral Variants Pipeline + +Every STM32 device has its own register map, but the peripherals inside them +are reused across the entire device range. This pipeline answers how many +*different* implementations of a peripheral actually exist by merging the +register maps of all peripheral instances of all devices into as few variants +as possible, see `modm_data.svd2variants`. + +The peripheral instances are grouped by their CMSIS structure type, which the +[header to SVD pipeline](header2svd.md) writes into the `groupName` element, +so that `TIM1` and `TIM17` are compared with each other, but not with `SPI1`. + +Some registers belong to the device rather than to the peripheral and are left +out of the comparison, see `modm_data.svd2variants.model.IGNORED_REGISTERS`. +The timer option registers `OR`, `OR1`-`OR3`, `AF1`, `AF2` and `TISEL` route +device signals to the timer inputs, e.g. which comparator drives the break +input, so their layout follows the device family and would otherwise split the +timers into one variant per family. + +## What Makes Two Register Maps Different + +Neither the names nor the addresses alone can decide whether two register maps +describe the same silicon, see `modm_data.svd2variants.merge`: + +- ST renames registers and bit fields without changing the layout, most + visibly between the STM32H7x and STM32H7[RS]x devices. Comparing names alone + reports these as different implementations, which they are not. +- ST reuses layouts for entirely different registers. The STM32F1 `GPIO_CRL` + and the `GPIO_MODER` of every other device both start with two-bit fields at + the same positions, so comparing addresses alone merges the two GPIO + implementations into one, which they are not. + +Therefore every element is compared by both keys, but not symmetrically: + +- A **name at two different locations** is a *conflict*: no single driver can + address both maps, so they are different variants. This is what separates + the STM32F1 GPIO (`IDR` at `0x08`) from every other GPIO (`IDR` at `0x10`). +- A **location with two different names** is a *rename*: the silicon is the + same and every name is kept as an alias. +- A bit field that only grew into the reserved bits above it is an optional + extension instead of a conflict, since ST widens fields like the `IWDG_PR` + prescaler in place. Pass `--strict` to treat this as a conflict instead. + +Unrelated elements never share a name, so two more rules catch what the names +cannot: + +- Two **differently laid out bit fields that share bits** are a conflict, + unless they are alternates of each other within one register map. Without + this, the STM32L1 `COMP_CSR_OUTSEL[23:21]` and the STM32G0 + `COMP_CSR_BLANKING[24:20]` merge into one register, each an optional feature + of the other. +- A **rename only counts inside the same register**, meaning the registers + share a name or at least one identically named bit field. The STM32F373 + `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` share a bit, but + nothing else, so the bit was reused for something else. This mostly matters + for binary compatibility, since in source mode the register name is already + part of every location. + +Everything else is a *population* difference: an element that only exists in +some of the instances becomes an optional feature of the merged map. A feature +is not a single register or bit field, but all elements that appear in exactly +the same instances, since those are enabled and disabled together, for example +the clock enable bits in `RCC` of a GPIO port that only the larger packages +bond out. + +## Binary, Source and Similar Compatibility + +Many register maps differ only in *where* a register sits, while the register +itself is unchanged. Which of the two matters depends on how the hardware is +addressed, so the analysis runs in three modes: + +- `binary`: elements are located by the **address** of their register, so + moving a register to another offset is a conflict. This is what a driver + that pokes at fixed addresses needs. +- `source`: elements are located by the **name** of their register, so a + register may move as long as it keeps its name and its layout. This is what + code written against a CMSIS header needs, since the structure member + abstracts the offset away. +- `similar`: like `source`, but differences that only exist in the + documentation are tolerated, see `modm_data.svd2variants.merge.documentation_only`. + This is a heuristic for how many implementations are *essentially* the same. + +Neither mode is strictly coarser than the other. Source compatibility merges +the two GPIO implementations into one, because the STM32F1 `IDR` at `0x08` and +the `IDR` at `0x10` of every other device have the same bit fields. But it +also splits `COMP` further, because several devices have a `CSR` register with +the same name and a different layout, which only collides once the name is +what identifies the register. + +The `similar` mode tolerates a register declared with another width (the +STM32F0 declares `USART_RDR` as `uint16_t`), a bit field with the same name +over overlapping bits, and a bit field described as smaller bit fields. The last +one is only tolerated if the names are related, like `COMP_CSR_BLANKING` and +`COMP_CSR_COMPxBLANKING`, or if the smaller bit fields tile one edge of a bit +field of at least four bits, either as a value split into at least two parts +like `USART_BRR_BRR` into `DIV_MANTISSA` and `DIV_FRACTION`, or as flags that +take up at most a quarter of it like `TIM_CNT_UIFCPY` in the top bit of +`TIM_CNT_CNT`. Unrelated bit fields nest by coincidence surprisingly often, e.g. +the STM32F4 `FLASH_OPTCR1_nWRP[27:16]` inside the STM32F7 +`FLASH_OPTCR1_BOOT_ADD1[31:16]`, so anything else is still a conflict. + +```bash +# Only compare by register address +python3 -m modm_data.svd2variants --all --mode binary +``` + +## Naming the Features + +A feature is a set of elements that appear in exactly the same instances, which +is a mechanical definition and says nothing about what the feature *is*. Most +of them do share something lexically, since ST names and documents related +elements alike, so a name is derived from the shared name prefix, the shared +description phrase or the registers involved, see `modm_data.svd2variants.naming`. + +That misses the semantics though: the seventeen bit fields of the first +capture/compare channel of a timer share no name and no description, but they +are obviously "Channel 1". A small language model names those much better, so +the names are generated into a lookup table keyed by the *content* of the +feature, see `modm_data.svd2variants.annotate`. The table is stable across +runs, reviewable as a diff, and only features that actually changed need to be +asked again. Every feature is named by the first of these that works: + +1. the lookup table of `--names`, which the homepage and the CI download from + the gist, so that they never need a model, +2. a local model, by default Gemma 4 26B-A4B served by LM Studio at + `http://localhost:1234/v1`, which only new features are sent to, +3. the heuristic. + +The prompt matters more than the size of the model. Keeping the numbers of +similar elements as a range turns "High line interrupt masking" into "Lines 28 +to 30 interrupt masking", and asking for a different name per feature stops +four ADC features from all becoming "Multimode and dual mode support". +Conversely, naming the vague words to avoid produced *more* of them. + +```bash +# Name the features that are missing from the table with a local model and update it +python3 -m modm_data.svd2variants --all --names feature-names.json +# Name them with the claude command line tool instead +python3 -m modm_data.svd2variants --all --names feature-names.json --claude +# Use another local model or server +python3 -m modm_data.svd2variants --all --names feature-names.json \ + --endpoint http://localhost:1234/v1 --model qwen/qwen3-4b-2507 +# Only use the table, as the CI does +python3 -m modm_data.svd2variants --all --names feature-names.json --no-generate +``` + +The homepage and the CI workflow currently use the heuristic names only. + +## Selective Analysis + +The reports are written to `log/stmicro/variants//*.txt`. + +```bash +# Analyze only these peripheral groups +python3 -m modm_data.svd2variants --group "GPIO" --group "TIM" +# Analyze all peripheral groups and write the JSON files +python3 -m modm_data.svd2variants --all --output docs/src/variants/ +``` + +The results are also published on the homepage with the +[Peripheral Variant Explorer](https://gist.github.com/salkinium/12a18032caa303697c6583937f6fcd16), +which loads the summary of both modes from `variants.json` and one file per +peripheral group from `binary/` and `source/`. diff --git a/tools/make/common.mk b/tools/make/common.mk index 1957bed..f98995b 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -62,6 +62,12 @@ build-homepage: @python3 -m modm_data.header2svd.stmicro --all --output docs/src/svd/ @curl -fsSL --create-dirs -o docs/src/svd/index.html \ https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/index.html + @curl -fsSL --create-dirs -o ext/cache/svd2variants/feature-names-gemma.json \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/feature-names-gemma.json || true + @python3 -m modm_data.svd2variants --all --input docs/src/svd/ \ + --names ext/cache/svd2variants/feature-names-gemma.json --output docs/src/variants/ + @curl -fsSL --create-dirs -o docs/src/variants/index.html \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/variants.html @(cd docs && mkdocs build) From 0bb6018d280dc176b97783c9fe9ed3406345f967 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Wed, 16 Sep 2026 21:24:26 +0200 Subject: [PATCH 5/5] [ci] Fix PyPI publishing and racing docs deployment PyPI rejects packages with direct URL dependencies, so modm-pinout cannot be declared in the docs extra. The docs workflow installs it explicitly. Two docs deployments running concurrently race each other when pushing to the data.modm.io repository. Serialize them with a concurrency group and rebase the generated content onto the remote before pushing. --- .github/workflows/deploy-docs.yml | 9 +++++++++ pyproject.toml | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index e7fd5d9..e55d694 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -10,6 +10,10 @@ on: jobs: build-upload-docs: runs-on: ubuntu-latest + # Only one deployment at a time, otherwise the pushes to data.modm.io race. + concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -22,6 +26,7 @@ jobs: - name: Install modm-data with docs dependencies run: | pip3 install -r tools/requirements.txt -e ".[docs]" + pip3 install git+https://github.com/modm-ext/modm-pinout.git@main - name: Install ARM toolchain and sources for the SVD files run: | @@ -50,6 +55,10 @@ jobs: git config --global user.email "bot@modm.io" git config --global user.name "modm update bot" cd docs/data.modm.io + # The generated content replaces the repository wholesale, so rebase + # onto whatever is on the remote instead of merging into it. + git fetch origin main + git reset --soft origin/main git add -A git diff-index --quiet HEAD || git commit -m "Update" git push origin main diff --git a/pyproject.toml b/pyproject.toml index 84d97e1..e0e0fe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,11 +44,12 @@ dynamic = ["version"] "Changelog" = "https://github.com/modm-io/modm-data/blob/main/CHANGELOG.md" [project.optional-dependencies] +# Note: modm-pinout is not declared here, since PyPI rejects packages with +# direct URL dependencies. The docs workflow installs it separately. docs = [ "pdoc>=14,<15", "mkdocs>=1.5,<2", "mkdocs-material>=9.5,<10", - "modm-pinout @ git+https://github.com/modm-ext/modm-pinout.git@main", ] all = ["modm_data[docs]"]