Merge STM32 peripheral register maps into variants

.github/workflows/deploy-docs.yml

@@ -10,6 +10,10 @@ on:
 jobs:
   build-upload-docs:
     runs-on: ubuntu-latest
+    # Only one deployment at a time, otherwise the pushes to data.modm.io race.
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -22,6 +26,7 @@ jobs:
       - name: Install modm-data with docs dependencies
         run: |
           pip3 install -r tools/requirements.txt -e ".[docs]"
+          pip3 install git+https://github.com/modm-ext/modm-pinout.git@main
 
       - name: Install ARM toolchain and sources for the SVD files
         run: |
@@ -50,6 +55,10 @@ jobs:
           git config --global user.email "bot@modm.io"
           git config --global user.name "modm update bot"
           cd docs/data.modm.io
+          # The generated content replaces the repository wholesale, so rebase
+          # onto whatever is on the remote instead of merging into it.
+          git fetch origin main
+          git reset --soft origin/main
           git add -A
           git diff-index --quiet HEAD || git commit -m "Update"
           git push origin main

.github/workflows/test-svd.yml

@@ -54,14 +54,38 @@ jobs:
           path: svd-archive/
           retention-days: 7
 
-      - name: Zip SVD Files
+      - name: Analyze Peripheral Variants
+        run: |
+          curl -fsSL --create-dirs -o ext/cache/svd2variants/feature-names-gemma.json \
+            https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/feature-names-gemma.json || true
+          python3 -m modm_data.svd2variants --all --input svd-archive/ --names ext/cache/svd2variants/feature-names-gemma.json \
+            --no-generate --output variants-archive/
+
+      - name: Upload Variant Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: variant-logs
+          path: log/stmicro/variants
+          retention-days: 1
+
+      - name: Upload Peripheral Variants
+        uses: actions/upload-artifact@v4
+        with:
+          name: variants-archive
+          path: variants-archive/
+          retention-days: 7
+
+      - name: Zip SVD Files and Peripheral Variants
         if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
         run: |
           zip -r svd-archive.zip svd-archive/
+          zip -r variants-archive.zip variants-archive/
 
       - name: Upload to Github Release
         uses: softprops/action-gh-release@v2
         if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
         with:
           files: |
             svd-archive.zip
+            variants-archive.zip

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Unreleased
+
+- Merge the register maps of all STM32 peripheral instances into the smallest
+  set of binary, source and similar compatible variants and publish them with
+  the Peripheral Variant Explorer.
+- Add the register access to the SVD files reconstructed from the CMSIS headers.
+
 ## 0.0.4
 
 - Move the data extraction pipelines of all vendors from modm-devices.

README.md

@@ -56,7 +56,9 @@ flowchart LR
     B(HTML) -->|html| C
     C(Python) -->|owl| E
     D(CMSIS-SVD) -->|cmsis-svd| C
+    D -->|svd2variants| K
     E[OWL]
+    K[Peripheral Variants]
     F(CMSIS Header) -->|header2svd| D
     I(Open Pin Data) -->|cubemx| C
     G(CubeMX) -->|cubemx| C

docs/.gitignore

@@ -2,6 +2,7 @@
 src/api/
 src/pinout/
 src/svd/
+src/variants/
 src/index.md
 src/source/overview.md
 src/pipeline/overview.md

docs/mkdocs.yml

@@ -87,6 +87,8 @@ nav:
       - HTML to SVD: pipeline/html2svd.md
       #- HTML to OWL: pipeline/html2owl.md
       - Header to SVD: pipeline/header2svd.md
+      - SVD to Variants: pipeline/svd2variants.md
   - API Reference: api/modm_data.html
   - Pinouts: pinout/index.html
   - SVD Explorer: svd/index.html
+  - Peripheral Variants: variants/index.html

docs/src/pipeline/svd2variants.md

@@ -0,0 +1,163 @@
+# SVD to Peripheral Variants Pipeline
+
+Every STM32 device has its own register map, but the peripherals inside them
+are reused across the entire device range. This pipeline answers how many
+*different* implementations of a peripheral actually exist by merging the
+register maps of all peripheral instances of all devices into as few variants
+as possible, see `modm_data.svd2variants`.
+
+The peripheral instances are grouped by their CMSIS structure type, which the
+[header to SVD pipeline](header2svd.md) writes into the `groupName` element,
+so that `TIM1` and `TIM17` are compared with each other, but not with `SPI1`.
+
+Some registers belong to the device rather than to the peripheral and are left
+out of the comparison, see `modm_data.svd2variants.model.IGNORED_REGISTERS`.
+The timer option registers `OR`, `OR1`-`OR3`, `AF1`, `AF2` and `TISEL` route
+device signals to the timer inputs, e.g. which comparator drives the break
+input, so their layout follows the device family and would otherwise split the
+timers into one variant per family.
+
+## What Makes Two Register Maps Different
+
+Neither the names nor the addresses alone can decide whether two register maps
+describe the same silicon, see `modm_data.svd2variants.merge`:
+
+- ST renames registers and bit fields without changing the layout, most
+  visibly between the STM32H7x and STM32H7[RS]x devices. Comparing names alone
+  reports these as different implementations, which they are not.
+- ST reuses layouts for entirely different registers. The STM32F1 `GPIO_CRL`
+  and the `GPIO_MODER` of every other device both start with two-bit fields at
+  the same positions, so comparing addresses alone merges the two GPIO
+  implementations into one, which they are not.
+
+Therefore every element is compared by both keys, but not symmetrically:
+
+- A **name at two different locations** is a *conflict*: no single driver can
+  address both maps, so they are different variants. This is what separates
+  the STM32F1 GPIO (`IDR` at `0x08`) from every other GPIO (`IDR` at `0x10`).
+- A **location with two different names** is a *rename*: the silicon is the
+  same and every name is kept as an alias.
+- A bit field that only grew into the reserved bits above it is an optional
+  extension instead of a conflict, since ST widens fields like the `IWDG_PR`
+  prescaler in place. Pass `--strict` to treat this as a conflict instead.
+
+Unrelated elements never share a name, so two more rules catch what the names
+cannot:
+
+- Two **differently laid out bit fields that share bits** are a conflict,
+  unless they are alternates of each other within one register map. Without
+  this, the STM32L1 `COMP_CSR_OUTSEL[23:21]` and the STM32G0
+  `COMP_CSR_BLANKING[24:20]` merge into one register, each an optional feature
+  of the other.
+- A **rename only counts inside the same register**, meaning the registers
+  share a name or at least one identically named bit field. The STM32F373
+  `COMP_CSR_COMP1EN[0]` and the STM32H503 `COMP_SR_C1VAL[0]` share a bit, but
+  nothing else, so the bit was reused for something else. This mostly matters
+  for binary compatibility, since in source mode the register name is already
+  part of every location.
+
+Everything else is a *population* difference: an element that only exists in
+some of the instances becomes an optional feature of the merged map. A feature
+is not a single register or bit field, but all elements that appear in exactly
+the same instances, since those are enabled and disabled together, for example
+the clock enable bits in `RCC` of a GPIO port that only the larger packages
+bond out.
+
+## Binary, Source and Similar Compatibility
+
+Many register maps differ only in *where* a register sits, while the register
+itself is unchanged. Which of the two matters depends on how the hardware is
+addressed, so the analysis runs in three modes:
+
+- `binary`: elements are located by the **address** of their register, so
+  moving a register to another offset is a conflict. This is what a driver
+  that pokes at fixed addresses needs.
+- `source`: elements are located by the **name** of their register, so a
+  register may move as long as it keeps its name and its layout. This is what
+  code written against a CMSIS header needs, since the structure member
+  abstracts the offset away.
+- `similar`: like `source`, but differences that only exist in the
+  documentation are tolerated, see `modm_data.svd2variants.merge.documentation_only`.
+  This is a heuristic for how many implementations are *essentially* the same.
+
+Neither mode is strictly coarser than the other. Source compatibility merges
+the two GPIO implementations into one, because the STM32F1 `IDR` at `0x08` and
+the `IDR` at `0x10` of every other device have the same bit fields. But it
+also splits `COMP` further, because several devices have a `CSR` register with
+the same name and a different layout, which only collides once the name is
+what identifies the register.
+
+The `similar` mode tolerates a register declared with another width (the
+STM32F0 declares `USART_RDR` as `uint16_t`), a bit field with the same name
+over overlapping bits, and a bit field described as smaller bit fields. The last
+one is only tolerated if the names are related, like `COMP_CSR_BLANKING` and
+`COMP_CSR_COMPxBLANKING`, or if the smaller bit fields tile one edge of a bit
+field of at least four bits, either as a value split into at least two parts
+like `USART_BRR_BRR` into `DIV_MANTISSA` and `DIV_FRACTION`, or as flags that
+take up at most a quarter of it like `TIM_CNT_UIFCPY` in the top bit of
+`TIM_CNT_CNT`. Unrelated bit fields nest by coincidence surprisingly often, e.g.
+the STM32F4 `FLASH_OPTCR1_nWRP[27:16]` inside the STM32F7
+`FLASH_OPTCR1_BOOT_ADD1[31:16]`, so anything else is still a conflict.
+
+```bash
+# Only compare by register address
+python3 -m modm_data.svd2variants --all --mode binary
+```
+
+## Naming the Features
+
+A feature is a set of elements that appear in exactly the same instances, which
+is a mechanical definition and says nothing about what the feature *is*. Most
+of them do share something lexically, since ST names and documents related
+elements alike, so a name is derived from the shared name prefix, the shared
+description phrase or the registers involved, see `modm_data.svd2variants.naming`.
+
+That misses the semantics though: the seventeen bit fields of the first
+capture/compare channel of a timer share no name and no description, but they
+are obviously "Channel 1". A small language model names those much better, so
+the names are generated into a lookup table keyed by the *content* of the
+feature, see `modm_data.svd2variants.annotate`. The table is stable across
+runs, reviewable as a diff, and only features that actually changed need to be
+asked again. Every feature is named by the first of these that works:
+
+1. the lookup table of `--names`, which the homepage and the CI download from
+   the gist, so that they never need a model,
+2. a local model, by default Gemma 4 26B-A4B served by LM Studio at
+   `http://localhost:1234/v1`, which only new features are sent to,
+3. the heuristic.
+
+The prompt matters more than the size of the model. Keeping the numbers of
+similar elements as a range turns "High line interrupt masking" into "Lines 28
+to 30 interrupt masking", and asking for a different name per feature stops
+four ADC features from all becoming "Multimode and dual mode support".
+Conversely, naming the vague words to avoid produced *more* of them.
+
+```bash
+# Name the features that are missing from the table with a local model and update it
+python3 -m modm_data.svd2variants --all --names feature-names.json
+# Name them with the claude command line tool instead
+python3 -m modm_data.svd2variants --all --names feature-names.json --claude
+# Use another local model or server
+python3 -m modm_data.svd2variants --all --names feature-names.json \
+    --endpoint http://localhost:1234/v1 --model qwen/qwen3-4b-2507
+# Only use the table, as the CI does
+python3 -m modm_data.svd2variants --all --names feature-names.json --no-generate
+```
+
+The homepage and the CI workflow currently use the heuristic names only.
+
+## Selective Analysis
+
+The reports are written to `log/stmicro/variants/<mode>/*.txt`.
+
+```bash
+# Analyze only these peripheral groups
+python3 -m modm_data.svd2variants --group "GPIO" --group "TIM"
+# Analyze all peripheral groups and write the JSON files
+python3 -m modm_data.svd2variants --all --output docs/src/variants/
+```
+
+The results are also published on the homepage with the
+[Peripheral Variant Explorer](https://gist.github.com/salkinium/12a18032caa303697c6583937f6fcd16),
+which loads the summary of both modes from `variants.json` and one file per
+peripheral group from `binary/` and `source/`.

pyproject.toml

@@ -44,11 +44,12 @@ dynamic = ["version"]
 "Changelog" = "https://github.com/modm-io/modm-data/blob/main/CHANGELOG.md"
 
 [project.optional-dependencies]
+# Note: modm-pinout is not declared here, since PyPI rejects packages with
+# direct URL dependencies. The docs workflow installs it separately.
 docs = [
     "pdoc>=14,<15",
     "mkdocs>=1.5,<2",
     "mkdocs-material>=9.5,<10",
-    "modm-pinout @ git+https://github.com/modm-ext/modm-pinout.git@main",
 ]
 all = ["modm_data[docs]"]
 

src/modm_data/header2svd/extract.py

@@ -37,7 +37,7 @@
 
 _CMSIS_PATH = ext_path("arm/cmsis/CMSIS/Core/Include")
 _CACHE_PATH = cache_path("cmsis/header2svd")
-_VERSION = 2
+_VERSION = 3
 
 
 @dataclass
@@ -48,6 +48,10 @@ class Member:
     """Type name of the member, or `pointer` for pointer members."""
     length: str | None
     """The array length expression or `None` if the member is not an array."""
+    access: str = "read-write"
+    """`read-only` for members declared with `__I` or `__IM`, which expand to
+    `volatile const`. The `__O` qualifier expands to the same `volatile` as `__IO`,
+    so write-only members cannot be told apart, but ST does not use it."""
 
 
 @dataclass
@@ -152,6 +156,7 @@ def members(cls, name):
             if isinstance(mtype, ctypes.Array):
                 length = "".join(token.value for token in mtype.size.tokens)
                 mtype = mtype.array_of
+            access = "read-only" if getattr(mtype, "const", False) else "read-write"
             if isinstance(mtype, (ctypes.Pointer, ctypes.FunctionType)) or not hasattr(mtype, "typename"):
                 if member.name:
                     result.append(Member(member.name, "pointer", length))
@@ -171,7 +176,7 @@ def members(cls, name):
                 result.append(Member(member.name, subtype, length))
                 continue
             if member.name:
-                result.append(Member(member.name, segment.name, length))
+                result.append(Member(member.name, segment.name, length, access))
         # Anonymous unions and structures without a member name
         for cid, inner_cls in inner.items():
             if cid not in named:

src/modm_data/header2svd/stmicro/memory_map.py

@@ -509,20 +509,25 @@ def _bit_fields(register: _Register, fields: dict[str, str], bitfields: _BitFiel
         for position, width, index in bitfields.fields[macro]:
             split = len(bitfields.fields[macro]) > 1
             fname = f"{name}_{index}" if index else name
-            candidates.append((fname, position, width, macro, split))
+            candidates.append((fname, position, width, macro, split, name))
 
     def bits(position, width):
         return set(range(position, position + width))
 
-    # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0
+    # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0. The parent
+    # may be one part of a non-contiguous bit field, e.g. CRYP_CR_ALGOMODE_AES_KEY in bits 5:3 of CRYP_CR_ALGOMODE,
+    # which also uses bit 19, otherwise the values replace the bit field itself.
     result = []
     for candidate in candidates:
         cbits = bits(*candidate[1:3])
         parent = next(
             (
                 c
                 for c in candidates
-                if c is not candidate and candidate[0].startswith(c[0] + "_") and cbits <= bits(*c[1:3]) and not c[4]
+                if c is not candidate
+                and candidate[0] != c[0]
+                and candidate[0].startswith(c[5] + "_")
+                and cbits <= bits(*c[1:3])
             ),
             None,
         )
@@ -547,7 +552,7 @@ def bits(position, width):
     # Prefer non-alias and non-split bit fields in the order of definition
     result.sort(key=lambda c: (c[3] in bitfields.alias, c[4], bitfields.order.get(c[3], 0), c[1]))
     used, alternate = {}, []
-    for fname, position, width, macro, split in result:
+    for fname, position, width, macro, split, _name in result:
         overlap = next((used[b] for b in bits(position, width) if b in used), None)
         if overlap is not None and fname not in register.fields:
             # A narrow field at the edge of a much wider field, e.g. TIM_CNT_UIFCPY in TIM_CNT_CNT
@@ -769,6 +774,7 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) ->
     interrupts = _interrupts(data, {re.sub(r"_NS$", "", pname) for pname in peripherals}, report)
 
     device = Device(name or data.header.stem, compatible=data.defines[:1])
+    access = {(typedef, member.name): member.access for typedef, members in data.structs.items() for member in members}
     signatures, addresses = {}, {}
     for pname, (address, ptype, registers) in sorted(peripherals.items(), key=lambda p: (p[1][0], p[0])):
         # Registers of sub-instances with colliding names are prefixed with the sub-instance name
@@ -791,6 +797,8 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) ->
         # The non-secure instances use the plain name
         peripheral = Peripheral(re.sub(r"_NS$", "", pname), ptype, address, parent=device)
         peripheral.description = data.type_descriptions.get(ptype, "")
+        if ptype:
+            peripheral.group = re.sub(r"_?TypeDef$", "", ptype)
         peripheral.interrupts = interrupts.get(peripheral.name, [])
         if (alternate := addresses.setdefault(address, peripheral.name)) != peripheral.name:
             peripheral.alternate = alternate
@@ -800,6 +808,7 @@ def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) ->
         offsets = {}
         for register in registers:
             treg = Register(register.name, register.offset, register.size, parent=peripheral)
+            treg.access = access.get((register.type, register.member), "read-write")
             treg.description = data.member_descriptions.get((register.type, register.member), "")
             if register.dim:
                 treg.dim = register.dim