[html2pinout] Discovery kits and pinout generator links

docs/mkdocs.yml

@@ -92,4 +92,4 @@ nav:
   - Pinouts: pinout/index.html
   - SVD Explorer: svd/index.html
   - Peripheral Variants: variants/index.html
-  - NUCLEO Pinouts: boards/index.html
+  - Board Pinouts: boards/index.html

src/modm_data/html2pinout/__init__.py

@@ -4,8 +4,9 @@
 """
 # HTML to Board Pinout Pipeline
 
-Extracts the connector pinouts (ST morpho, ARDUINO, Zio) of the NUCLEO boards
-from the connector tables of their user manuals.
+Extracts the connector pinouts (ST morpho, ARDUINO, Zio, STMod+, Pmod, extension)
+of the NUCLEO boards and Discovery kits from the connector tables of their user
+manuals.
 """
 
 import re
@@ -18,15 +19,22 @@
 
 LOGGER = logging.getLogger(__name__)
 
-_TABLES = r"morpho|arduino|zio"
-_NUMBER = re.compile(r"^(zio|connector)?pin(number|no\.?|nbr)?$")
+_TABLES = r"morpho|arduino|zio|stmod|pmod|mikrobus|extension"
+# The daughterboard connectors of the evaluation boards are unique to each board
+_SKIPPED = r"daughter ?board|shield"
+_NUMBER = re.compile(r"^(zio|connector)?pin(number|no\.?|nbr)?(\((cn|p)\d+\))?$")
 _CONNECTOR = re.compile(r"^(connector|cn)$")
 _MCU = re.compile(r"(stm32|mcu|stlink).*(pin?|port|name)$|^port$")
 _GPIO = re.compile(r"(?<![A-Z0-9])P[A-Q]\d{1,2}(?!\d)")
 _BRIDGE = re.compile(r"\b(?:SB|JP)\d+\b")
 _STLINK = re.compile(r"(?:P[A-K]\d+\W+(?:and\W+)?)+(?:on|of)\W+(?:the\W+)?ST-?LINK")
 _CN = re.compile(r"\bCN\d+\b")
-_BOARD = re.compile(r"NUCLEO-(?:[FGLHUCN]|WBA?|WL)\d(?![0-9A-Z]*XX)[0-9A-Z]*(?:-[PQ])?\b")
+_NAME = re.compile(r"\b(?:CN|P)\d+\b")
+_BOARD = re.compile(
+    r"NUCLEO-(?:[FGLHUCN]|WBA?|WL)\d(?![0-9A-Z]*XX)[0-9A-Z]*(?:-[PQ])?\b"
+    r"|\bSTM32[A-Z0-9]*-?(?:DISCO(?:VERY)?|DISC\d|DK\d?|EV(?:AL)?\d?)\b"
+    r"|\bB-[A-Z0-9]+-[A-Z0-9]+\b"
+)
 
 # The manuals spell the same signal in many ways
 _SIGNALS = [
@@ -91,7 +99,8 @@ def _field(header: str) -> str:
 
 
 def _boards(html: str) -> set[str]:
-    return set(_BOARD.findall(html))
+    # The Discovery kits also have a marketing name: 32F429IDISCOVERY is the STM32F429I-DISCO
+    return set(_BOARD.findall(re.sub(r"\b32([A-Z]\d\w+?)DISCOVERY\b", r"STM32\1-DISCO", html)))
 
 
 def _document_boards(chapters: list) -> set[str]:
@@ -133,17 +142,19 @@ def _table_pinout(table, kind: str) -> dict[str, dict]:
     rows = range(table._hrows, table.rows)
     if not (groups := _groups(header)):
         # Some tables only name the connectors in the header, then the pin numbers are found by their content
-        digits = [sum(_text(table.cell(x, y)).isdigit() for y in rows) for x in range(table.columns)]
+        cells = [[_text(table.cell(x, y)) for y in rows] for x in range(table.columns)]
+        # Repeated numbers are the values of a configuration table instead
+        digits = [sum(c.isdigit() for c in column) * (len(set(column)) > len(column) / 2) for column in cells]
         hy, header = table._hrows, ["Pin" if d > len(rows) * 0.8 else "Pin name" for d in digits]
         groups = _groups(header)
-    captions = set(_CN.findall(table.caption()))
+    captions = set(_NAME.findall(table.caption()))
     pinout = defaultdict(dict)
     unnamed = 1
 
     for number, columns in groups.items():
         # The connector is named in the header above, in its own column or in the caption
-        above = " ".join(_text(table.cell(number, y)) for y in range(hy))
-        connector = next(iter(_CN.findall(above)), next(iter(captions), "") if len(captions) == 1 else "")
+        above = " ".join(_text(table.cell(number, y)) for y in range(hy + 1))
+        connector = next(iter(_NAME.findall(above)), next(iter(captions), "") if len(captions) == 1 else "")
         connector_columns = [
             c
             for c in columns
@@ -185,6 +196,49 @@ def _table_pinout(table, kind: str) -> dict[str, dict]:
     return {c: pins for c, pins in pinout.items() if pins}
 
 
+def _device_table_pinout(table) -> dict[str, dict]:
+    """
+    The older Discovery kits have one table of all device pins instead of one per connector: one row per device
+    pin, with one column per board function and per extension connector (`P1`, `P2`), which contains the
+    connector pin numbers.
+    """
+    if table._hrows < 2:
+        return {}
+    header = [_text(table.cell(x, table._hrows - 1)) for x in range(table.columns)]
+    keys = [_key(h) for h in header]
+    if "mainfunction" not in keys:
+        return {}
+    main = keys.index("mainfunction")
+    power = keys.index("powersupply") if "powersupply" in keys else main
+    connectors = [x for x, h in enumerate(header) if re.fullmatch(r"P\d", h)]
+    functions = [
+        x
+        for x in range(table.columns)
+        if _key(_text(table.cell(x, 0))).startswith("boardfunction")
+        and x not in connectors
+        and keys[x] not in {"freei/o", "powersupply"}
+        and not _CN.fullmatch(header[x])
+    ]
+    pinout = defaultdict(dict)
+    for y in range(table._hrows, table.rows):
+        # The supplies are not device pins
+        name = _text(table.cell(main, y))
+        name = _text(table.cell(power, y)) if name in {"", "-"} else name
+        signal, kind = _signal(name)
+        texts = [(header[x], _text(table.cell(x, y))) for x in functions]
+        function = " / ".join(f"{h}: {text}" for h, text in texts if text not in {"", "-"})
+        for x in connectors:
+            for number in re.findall(r"\d+", _text(table.cell(x, y))):
+                pinout[header[x]][int(number)] = {
+                    "signal": signal,
+                    "kind": kind,
+                    "pin": name,
+                    **({"function": function} if function else {}),
+                }
+    # P1 is the left connector
+    return dict(sorted(pinout.items()))
+
+
 def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, dict]]:
     """
     :return: `board -> connector -> {type, side, pins: number -> {signal, kind, ...}}` for all NUCLEO boards in
@@ -201,7 +255,12 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di
             kind = re.search(_TABLES, table.caption(), re.IGNORECASE) or re.search(
                 _TABLES, table.heading(), re.IGNORECASE
             )
-            if not kind or not (pinout := _table_pinout(table, kind.group(0).lower())):
+            if not kind or re.search(_SKIPPED, table.caption() + table.heading(), re.IGNORECASE):
+                continue
+            kind = kind.group(0).lower()
+            if pinout := _device_table_pinout(table):
+                kind = "extension"
+            elif not (pinout := _table_pinout(table, kind)):
                 continue
             # Some manuals describe several boards with one table for each
             for board in _boards(table.caption()) or boards:
@@ -211,7 +270,7 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di
                     if sum(n in known and known[n]["signal"] != p["signal"] for n, p in pins.items()) > len(pins) / 2:
                         LOGGER.warning(f"{document.fullname}: {board} {connector} is redefined in '{table.caption()}'")
                         connector += "-2"
-                    current = pinouts[board].setdefault(connector, {"type": kind.group(0).lower(), "pins": {}})["pins"]
+                    current = pinouts[board].setdefault(connector, {"type": kind, "pins": {}})["pins"]
                     for number, pin in pins.items():
                         if number in current and current[number]["signal"] != pin["signal"]:
                             LOGGER.warning(
@@ -231,6 +290,10 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di
                 pins.setdefault(number, {"signal": signal, "kind": kind, "label": name, "remark": f"From {source}"})
 
     for connectors in pinouts.values():
+        # A table without a connector name describes the named connector of the same type once more
+        named = {c["type"] for name, c in connectors.items() if not name.startswith(c["type"])}
+        for name in [n for n, c in connectors.items() if n.startswith(c["type"]) and c["type"] in named]:
+            del connectors[name]
         for connector in connectors.values():
             connector["pins"] = dict(sorted(connector["pins"].items()))
         # The manuals list the left morpho connector first

src/modm_data/html2pinout/__main__.py

@@ -13,6 +13,7 @@
     inherit_bridge_defaults,
 )
 from modm_data.html2pinout.signals import board_uses, device_for_board, device_signals, has_board_file
+from modm_data.html2pinout.share import pinout_link
 from modm_data.utils import ext_path
 
 
@@ -25,6 +26,7 @@ def main():
     parser.add_argument("--document", type=Path, help="A single user manual, otherwise all are converted.")
     parser.add_argument("--input", type=Path, default=ext_path("stmicro/html-archive"))
     parser.add_argument("--output", type=Path, default=ext_path("stmicro/pinout/nucleo.json"))
+    parser.add_argument("--pinout", type=Path, help="The output folder of modm_pinout to link the boards to.")
     args = parser.parse_args()
     logging.basicConfig(level=logging.WARNING, format="%(message)s")
 
@@ -39,14 +41,17 @@ def main():
                 latest[name] = path
         paths = sorted(latest.values())
 
-    pinouts = {}
+    pinouts, scores = {}, {}
     for path in paths:
         document = Document(path.absolute())
-        for board, connectors in board_pinouts_from_user_manual(document).items():
-            pins = sum(len(c["pins"]) for c in connectors.values())
-            # Several manuals may name a board, the one that describes the most of it is its own
-            if board in pinouts and pins <= sum(len(c["pins"]) for c in pinouts[board]["connectors"].values()):
+        boards = board_pinouts_from_user_manual(document)
+        for board, connectors in boards.items():
+            # Several manuals may name a board, the one that describes the most of it is its own. The manuals
+            # of the Discovery kits however also name their predecessors, which have their own smaller manual.
+            score = (board.startswith("NUCLEO") or -len(boards), sum(len(c["pins"]) for c in connectors.values()))
+            if board in pinouts and score <= scores[board]:
                 continue
+            scores[board] = score
             pinouts[board] = {"board": board, "document": document.fullname, "connectors": connectors}
         bridges = board_bridges_from_user_manual(document)
         # Older revisions of the manual may still know the defaults that this one lost
@@ -65,6 +70,9 @@ def main():
         # NUCLEO-WL55JC1. A real board like NUCLEO-L452RE next to NUCLEO-L452RE-P has its own description.
         if any(other != board and other.startswith(board) for other in pinouts) and not has_board_file(board):
             continue
+        # The Discovery kits have marketing names and placeholders next to their real name
+        if not board.startswith("NUCLEO") and not has_board_file(board):
+            continue
         # Only the signals of the GPIOs on the connectors are of interest
         gpios = {p["signal"] for c in data["connectors"].values() for p in c["pins"].values() if p["kind"] == "gpio"}
         device = device_for_board(board)
@@ -74,9 +82,14 @@ def main():
         # Only the bridges that mention a GPIO on the connectors
         bridges = [dict(b, gpios=[g for g in b["gpios"] if g in gpios]) for b in data.pop("bridges", [])]
         data["bridges"] = [b for b in bridges if b["gpios"]]
-        data["uses"] = {gpio: use for gpio, use in board_uses(board).items() if gpio in gpios}
+        uses = board_uses(board)
+        data["uses"] = {gpio: use for gpio, use in uses.items() if gpio in gpios}
+        if device and args.pinout and (link := pinout_link(args.pinout, device[0], data["connectors"], uses)):
+            data["pinout"] = link
         boards.append(data)
-        sizes = ", ".join(f"{c}={len(p['pins'])}" for c, p in data["connectors"].items() if p["type"] == "morpho")
+        sizes = ", ".join(
+            f"{c}={len(p['pins'])}" for c, p in data["connectors"].items() if p["type"] in {"morpho", "extension"}
+        )
         print(f"{board:20} {data['document']:12} {data['device'] or '-':16} {sizes}")
 
     if not boards:

src/modm_data/html2pinout/share.py

@@ -0,0 +1,86 @@
+# Copyright 2026, Niklas Hauser
+# SPDX-License-Identifier: MPL-2.0
+
+"""
+Links a board to the page of its device in the pinout generator of `modm_pinout`,
+so that the generator starts with what the board already uses the pins for.
+"""
+
+import re
+import json
+import zlib
+import base64
+from functools import cache
+from pathlib import Path
+
+
+@cache
+def _json(path: Path) -> dict:
+    return json.loads(path.read_text())
+
+
+def _function(signal: str, functions: list[str]) -> str | None:
+    # CubeMX calls it SYS_JTMS-SWDIO and ETH_RMII_TXD0, the generator SYS_JTMS/SYS_SWDIO and ETH_TXD0
+    names = {signal, signal.split("-")[0], re.sub(r"_R?MII", "", signal), re.sub(r"^S_", "", signal)}
+    return next((f for f in functions if names & set(f.split("/"))), None)
+
+
+def share_state(rows: list[dict], connectors: dict, uses: dict) -> dict:
+    """
+    :param rows: The pins of the device in the generator: `{row_id, short_name, functions}`.
+    :param connectors: The connectors of the board.
+    :param uses: What the board itself uses the GPIOs for.
+    :return: The state of the generator with the functions used by the board selected, and everything else that
+             is known about a pin as its comment.
+    """
+    comments = {}
+    for name, connector in connectors.items():
+        for number, pin in connector["pins"].items():
+            if pin["kind"] == "gpio":
+                # The ARDUINO pin names are better known than their position
+                label = pin.get("label", "")
+                label = f" ({label})" if re.fullmatch(r"[AD]\d+", label) else ""
+                comments.setdefault(pin["signal"], []).append(f"{name}-{number}{label}")
+
+    selected, names = {}, {}
+    for row in rows:
+        # The generator names the pins PA0/WKUP or PC14/OSC32_IN
+        gpio = row["short_name"].split("/")[0]
+        use = uses.get(gpio, {})
+        function = _function(use.get("signal", ""), row["functions"])
+        if function:
+            selected[str(row["row_id"])] = [function]
+        label = " ".join(filter(None, [use.get("label"), use.get("part") and f"[{use['part']}]"]))
+        # A function that the generator does not know is still worth knowing
+        label = label or ("" if function else use.get("signal", ""))
+        if comment := ", ".join(filter(None, [label, *comments.get(gpio, [])])):
+            names[str(row["row_id"])] = comment
+    return {"version": 1, "selectedByRowId": selected, "namesByRowId": names}
+
+
+def pinout_link(path: Path, device: str, connectors: dict, uses: dict) -> str | None:
+    """
+    :param path: The output folder of `modm_pinout`.
+    :param device: The name of the device on the board like `STM32F429ZITx`.
+    :return: The page of the device relative to the folder with the state of the board as shared URL, or `None`
+             if the generator does not know the device.
+    """
+    if not (path / "manifest.json").exists():
+        return None
+    # The x is the temperature range, of which the first one is as good as any other. The devices with several
+    # cores have one page for each core, of which the last one is the main core.
+    pattern = re.escape(device.lower()).replace("x", r"\w") + r"(@m\d+)?"
+    chips = [c for c in _json(path / "manifest.json")["devices"] if re.fullmatch(pattern, c["chip_id"])]
+    if not chips:
+        return None
+    chip = max(chips, key=lambda c: (-ord(c["chip_id"][len(device) - 1]), c["chip_id"]))
+    rows = _json(path / chip["data"])["devices"][chip["chip_id"]]["rows"]
+    share = {
+        "version": 1,
+        "format": "modm-pinout-share",
+        "chipId": chip["chip_id"],
+        "state": share_state(rows, connectors, uses),
+    }
+    deflate = zlib.compressobj(9, zlib.DEFLATED, -15)
+    data = deflate.compress(json.dumps(share, separators=(",", ":")).encode()) + deflate.flush()
+    return f"{chip['page']}#share=v1d.{base64.urlsafe_b64encode(data).decode().rstrip('=')}"

src/modm_data/html2pinout/signals.py

@@ -48,7 +48,10 @@ def device_for_board(board: str) -> tuple[str, str] | None:
     :return: The name of the mounted device and of its CubeMX description, or `None` if unknown.
     """
     if not (match := re.fullmatch(r"NUCLEO-(\w+?)(?:-([PQ]))?", board)):
-        return None
+        # The names of the other boards do not contain the whole device name, but their board file does
+        mcu = dict(re.findall(r"^Mcu\.(Name|UserName)=(.*)$", _board_file(board), re.MULTILINE))
+        exists = mcu and (_mcu_path() / f"{mcu['Name']}.xml").exists()
+        return (mcu["UserName"], mcu["Name"]) if exists else None
     smps = match.group(2) or ""
     # Some board names end in a revision digit, which is not part of the device name
     for part in ("STM32" + match.group(1), "STM32" + match.group(1)[:-1]):
@@ -89,7 +92,13 @@ def device_signals(name: str) -> dict[str, dict[str, int | None]]:
 
 
 def _board_files(board: str) -> list[Path]:
-    return sorted((_cubemx_path() / "plugins/boardmanager/boards").glob(f"*_Nucleo_{board}_*_Board_AllConfig.ioc"))
+    return sorted((_cubemx_path() / "plugins/boardmanager/boards").glob(f"*_{board}_*_Board_AllConfig.ioc"))
+
+
+def _board_file(board: str) -> str:
+    # The shortest name is the default configuration without TrustZone or multi-core variants
+    paths = _board_files(board)
+    return min(paths, key=lambda p: len(p.name)).read_text(errors="replace") if paths else ""
 
 
 def has_board_file(board: str) -> bool:
@@ -103,11 +112,8 @@ def board_uses(board: str) -> dict[str, dict[str, str]]:
     :return: `gpio -> {use, label, part, signal}` for all GPIOs that are connected to something on the board
              itself, like the ST-LINK, an Ethernet PHY, a crystal, LEDs or buttons.
     """
-    if not (paths := _board_files(board)):
-        return {}
     pins = {}
-    # The shortest name is the default configuration without TrustZone or multi-core variants
-    for line in min(paths, key=lambda p: len(p.name)).read_text(errors="replace").splitlines():
+    for line in _board_file(board).splitlines():
         if match := re.match(r"(P[A-Z]\d+)[^.=]*\.(GPIO_Label|Signal)=(.*)", line):
             pins.setdefault(match.group(1), {})[match.group(2)] = match.group(3).strip()