From f7c58bc7b5adc86e54624879a808b3f767104964 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Sat, 3 Oct 2026 21:58:06 +0200 Subject: [PATCH] [html2pinout] Add the Discovery kits and link to the pinout generator The STMod+, Pmod, mikroBUS and extension connectors of the Discovery kits and evaluation boards are extracted in addition to the NUCLEO boards. The older Discovery kits describe their extension connectors as one table of all device pins, which is parsed separately. Only boards known to CubeMX are kept, which removes their marketing names and placeholders, and the device is taken from the CubeMX board description. Every board links to the page of its device in the pinout generator, with the functions that the board already uses selected and the board labels and connector positions as comments. --- docs/mkdocs.yml | 2 +- src/modm_data/html2pinout/__init__.py | 87 +++++++++++++++++++++++---- src/modm_data/html2pinout/__main__.py | 27 ++++++--- src/modm_data/html2pinout/share.py | 86 ++++++++++++++++++++++++++ src/modm_data/html2pinout/signals.py | 18 ++++-- test/html2pinout_test.py | 73 +++++++++++++++++++++- tools/make/common.mk | 2 +- 7 files changed, 267 insertions(+), 28 deletions(-) create mode 100644 src/modm_data/html2pinout/share.py diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4b8af23..cd4e18b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -92,4 +92,4 @@ nav: - Pinouts: pinout/index.html - SVD Explorer: svd/index.html - Peripheral Variants: variants/index.html - - NUCLEO Pinouts: boards/index.html + - Board Pinouts: boards/index.html diff --git a/src/modm_data/html2pinout/__init__.py b/src/modm_data/html2pinout/__init__.py index 258b70e..f0dd8b0 100644 --- a/src/modm_data/html2pinout/__init__.py +++ b/src/modm_data/html2pinout/__init__.py @@ -4,8 +4,9 @@ """ # HTML to Board Pinout Pipeline -Extracts the connector pinouts (ST morpho, ARDUINO, Zio) of the NUCLEO boards -from the connector tables of their user manuals. +Extracts the connector pinouts (ST morpho, ARDUINO, Zio, STMod+, Pmod, extension) +of the NUCLEO boards and Discovery kits from the connector tables of their user +manuals. """ import re @@ -18,15 +19,22 @@ LOGGER = logging.getLogger(__name__) -_TABLES = r"morpho|arduino|zio" -_NUMBER = re.compile(r"^(zio|connector)?pin(number|no\.?|nbr)?$") +_TABLES = r"morpho|arduino|zio|stmod|pmod|mikrobus|extension" +# The daughterboard connectors of the evaluation boards are unique to each board +_SKIPPED = r"daughter ?board|shield" +_NUMBER = re.compile(r"^(zio|connector)?pin(number|no\.?|nbr)?(\((cn|p)\d+\))?$") _CONNECTOR = re.compile(r"^(connector|cn)$") _MCU = re.compile(r"(stm32|mcu|stlink).*(pin?|port|name)$|^port$") _GPIO = re.compile(r"(? str: def _boards(html: str) -> set[str]: - return set(_BOARD.findall(html)) + # The Discovery kits also have a marketing name: 32F429IDISCOVERY is the STM32F429I-DISCO + return set(_BOARD.findall(re.sub(r"\b32([A-Z]\d\w+?)DISCOVERY\b", r"STM32\1-DISCO", html))) def _document_boards(chapters: list) -> set[str]: @@ -133,17 +142,19 @@ def _table_pinout(table, kind: str) -> dict[str, dict]: rows = range(table._hrows, table.rows) if not (groups := _groups(header)): # Some tables only name the connectors in the header, then the pin numbers are found by their content - digits = [sum(_text(table.cell(x, y)).isdigit() for y in rows) for x in range(table.columns)] + cells = [[_text(table.cell(x, y)) for y in rows] for x in range(table.columns)] + # Repeated numbers are the values of a configuration table instead + digits = [sum(c.isdigit() for c in column) * (len(set(column)) > len(column) / 2) for column in cells] hy, header = table._hrows, ["Pin" if d > len(rows) * 0.8 else "Pin name" for d in digits] groups = _groups(header) - captions = set(_CN.findall(table.caption())) + captions = set(_NAME.findall(table.caption())) pinout = defaultdict(dict) unnamed = 1 for number, columns in groups.items(): # The connector is named in the header above, in its own column or in the caption - above = " ".join(_text(table.cell(number, y)) for y in range(hy)) - connector = next(iter(_CN.findall(above)), next(iter(captions), "") if len(captions) == 1 else "") + above = " ".join(_text(table.cell(number, y)) for y in range(hy + 1)) + connector = next(iter(_NAME.findall(above)), next(iter(captions), "") if len(captions) == 1 else "") connector_columns = [ c for c in columns @@ -185,6 +196,49 @@ def _table_pinout(table, kind: str) -> dict[str, dict]: return {c: pins for c, pins in pinout.items() if pins} +def _device_table_pinout(table) -> dict[str, dict]: + """ + The older Discovery kits have one table of all device pins instead of one per connector: one row per device + pin, with one column per board function and per extension connector (`P1`, `P2`), which contains the + connector pin numbers. + """ + if table._hrows < 2: + return {} + header = [_text(table.cell(x, table._hrows - 1)) for x in range(table.columns)] + keys = [_key(h) for h in header] + if "mainfunction" not in keys: + return {} + main = keys.index("mainfunction") + power = keys.index("powersupply") if "powersupply" in keys else main + connectors = [x for x, h in enumerate(header) if re.fullmatch(r"P\d", h)] + functions = [ + x + for x in range(table.columns) + if _key(_text(table.cell(x, 0))).startswith("boardfunction") + and x not in connectors + and keys[x] not in {"freei/o", "powersupply"} + and not _CN.fullmatch(header[x]) + ] + pinout = defaultdict(dict) + for y in range(table._hrows, table.rows): + # The supplies are not device pins + name = _text(table.cell(main, y)) + name = _text(table.cell(power, y)) if name in {"", "-"} else name + signal, kind = _signal(name) + texts = [(header[x], _text(table.cell(x, y))) for x in functions] + function = " / ".join(f"{h}: {text}" for h, text in texts if text not in {"", "-"}) + for x in connectors: + for number in re.findall(r"\d+", _text(table.cell(x, y))): + pinout[header[x]][int(number)] = { + "signal": signal, + "kind": kind, + "pin": name, + **({"function": function} if function else {}), + } + # P1 is the left connector + return dict(sorted(pinout.items())) + + def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, dict]]: """ :return: `board -> connector -> {type, side, pins: number -> {signal, kind, ...}}` for all NUCLEO boards in @@ -201,7 +255,12 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di kind = re.search(_TABLES, table.caption(), re.IGNORECASE) or re.search( _TABLES, table.heading(), re.IGNORECASE ) - if not kind or not (pinout := _table_pinout(table, kind.group(0).lower())): + if not kind or re.search(_SKIPPED, table.caption() + table.heading(), re.IGNORECASE): + continue + kind = kind.group(0).lower() + if pinout := _device_table_pinout(table): + kind = "extension" + elif not (pinout := _table_pinout(table, kind)): continue # Some manuals describe several boards with one table for each for board in _boards(table.caption()) or boards: @@ -211,7 +270,7 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di if sum(n in known and known[n]["signal"] != p["signal"] for n, p in pins.items()) > len(pins) / 2: LOGGER.warning(f"{document.fullname}: {board} {connector} is redefined in '{table.caption()}'") connector += "-2" - current = pinouts[board].setdefault(connector, {"type": kind.group(0).lower(), "pins": {}})["pins"] + current = pinouts[board].setdefault(connector, {"type": kind, "pins": {}})["pins"] for number, pin in pins.items(): if number in current and current[number]["signal"] != pin["signal"]: LOGGER.warning( @@ -231,6 +290,10 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di pins.setdefault(number, {"signal": signal, "kind": kind, "label": name, "remark": f"From {source}"}) for connectors in pinouts.values(): + # A table without a connector name describes the named connector of the same type once more + named = {c["type"] for name, c in connectors.items() if not name.startswith(c["type"])} + for name in [n for n, c in connectors.items() if n.startswith(c["type"]) and c["type"] in named]: + del connectors[name] for connector in connectors.values(): connector["pins"] = dict(sorted(connector["pins"].items())) # The manuals list the left morpho connector first diff --git a/src/modm_data/html2pinout/__main__.py b/src/modm_data/html2pinout/__main__.py index 5abf93e..34fb9dd 100644 --- a/src/modm_data/html2pinout/__main__.py +++ b/src/modm_data/html2pinout/__main__.py @@ -13,6 +13,7 @@ inherit_bridge_defaults, ) from modm_data.html2pinout.signals import board_uses, device_for_board, device_signals, has_board_file +from modm_data.html2pinout.share import pinout_link from modm_data.utils import ext_path @@ -25,6 +26,7 @@ def main(): parser.add_argument("--document", type=Path, help="A single user manual, otherwise all are converted.") parser.add_argument("--input", type=Path, default=ext_path("stmicro/html-archive")) parser.add_argument("--output", type=Path, default=ext_path("stmicro/pinout/nucleo.json")) + parser.add_argument("--pinout", type=Path, help="The output folder of modm_pinout to link the boards to.") args = parser.parse_args() logging.basicConfig(level=logging.WARNING, format="%(message)s") @@ -39,14 +41,17 @@ def main(): latest[name] = path paths = sorted(latest.values()) - pinouts = {} + pinouts, scores = {}, {} for path in paths: document = Document(path.absolute()) - for board, connectors in board_pinouts_from_user_manual(document).items(): - pins = sum(len(c["pins"]) for c in connectors.values()) - # Several manuals may name a board, the one that describes the most of it is its own - if board in pinouts and pins <= sum(len(c["pins"]) for c in pinouts[board]["connectors"].values()): + boards = board_pinouts_from_user_manual(document) + for board, connectors in boards.items(): + # Several manuals may name a board, the one that describes the most of it is its own. The manuals + # of the Discovery kits however also name their predecessors, which have their own smaller manual. + score = (board.startswith("NUCLEO") or -len(boards), sum(len(c["pins"]) for c in connectors.values())) + if board in pinouts and score <= scores[board]: continue + scores[board] = score pinouts[board] = {"board": board, "document": document.fullname, "connectors": connectors} bridges = board_bridges_from_user_manual(document) # Older revisions of the manual may still know the defaults that this one lost @@ -65,6 +70,9 @@ def main(): # NUCLEO-WL55JC1. A real board like NUCLEO-L452RE next to NUCLEO-L452RE-P has its own description. if any(other != board and other.startswith(board) for other in pinouts) and not has_board_file(board): continue + # The Discovery kits have marketing names and placeholders next to their real name + if not board.startswith("NUCLEO") and not has_board_file(board): + continue # Only the signals of the GPIOs on the connectors are of interest gpios = {p["signal"] for c in data["connectors"].values() for p in c["pins"].values() if p["kind"] == "gpio"} device = device_for_board(board) @@ -74,9 +82,14 @@ def main(): # Only the bridges that mention a GPIO on the connectors bridges = [dict(b, gpios=[g for g in b["gpios"] if g in gpios]) for b in data.pop("bridges", [])] data["bridges"] = [b for b in bridges if b["gpios"]] - data["uses"] = {gpio: use for gpio, use in board_uses(board).items() if gpio in gpios} + uses = board_uses(board) + data["uses"] = {gpio: use for gpio, use in uses.items() if gpio in gpios} + if device and args.pinout and (link := pinout_link(args.pinout, device[0], data["connectors"], uses)): + data["pinout"] = link boards.append(data) - sizes = ", ".join(f"{c}={len(p['pins'])}" for c, p in data["connectors"].items() if p["type"] == "morpho") + sizes = ", ".join( + f"{c}={len(p['pins'])}" for c, p in data["connectors"].items() if p["type"] in {"morpho", "extension"} + ) print(f"{board:20} {data['document']:12} {data['device'] or '-':16} {sizes}") if not boards: diff --git a/src/modm_data/html2pinout/share.py b/src/modm_data/html2pinout/share.py new file mode 100644 index 0000000..a9da372 --- /dev/null +++ b/src/modm_data/html2pinout/share.py @@ -0,0 +1,86 @@ +# Copyright 2026, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +Links a board to the page of its device in the pinout generator of `modm_pinout`, +so that the generator starts with what the board already uses the pins for. +""" + +import re +import json +import zlib +import base64 +from functools import cache +from pathlib import Path + + +@cache +def _json(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _function(signal: str, functions: list[str]) -> str | None: + # CubeMX calls it SYS_JTMS-SWDIO and ETH_RMII_TXD0, the generator SYS_JTMS/SYS_SWDIO and ETH_TXD0 + names = {signal, signal.split("-")[0], re.sub(r"_R?MII", "", signal), re.sub(r"^S_", "", signal)} + return next((f for f in functions if names & set(f.split("/"))), None) + + +def share_state(rows: list[dict], connectors: dict, uses: dict) -> dict: + """ + :param rows: The pins of the device in the generator: `{row_id, short_name, functions}`. + :param connectors: The connectors of the board. + :param uses: What the board itself uses the GPIOs for. + :return: The state of the generator with the functions used by the board selected, and everything else that + is known about a pin as its comment. + """ + comments = {} + for name, connector in connectors.items(): + for number, pin in connector["pins"].items(): + if pin["kind"] == "gpio": + # The ARDUINO pin names are better known than their position + label = pin.get("label", "") + label = f" ({label})" if re.fullmatch(r"[AD]\d+", label) else "" + comments.setdefault(pin["signal"], []).append(f"{name}-{number}{label}") + + selected, names = {}, {} + for row in rows: + # The generator names the pins PA0/WKUP or PC14/OSC32_IN + gpio = row["short_name"].split("/")[0] + use = uses.get(gpio, {}) + function = _function(use.get("signal", ""), row["functions"]) + if function: + selected[str(row["row_id"])] = [function] + label = " ".join(filter(None, [use.get("label"), use.get("part") and f"[{use['part']}]"])) + # A function that the generator does not know is still worth knowing + label = label or ("" if function else use.get("signal", "")) + if comment := ", ".join(filter(None, [label, *comments.get(gpio, [])])): + names[str(row["row_id"])] = comment + return {"version": 1, "selectedByRowId": selected, "namesByRowId": names} + + +def pinout_link(path: Path, device: str, connectors: dict, uses: dict) -> str | None: + """ + :param path: The output folder of `modm_pinout`. + :param device: The name of the device on the board like `STM32F429ZITx`. + :return: The page of the device relative to the folder with the state of the board as shared URL, or `None` + if the generator does not know the device. + """ + if not (path / "manifest.json").exists(): + return None + # The x is the temperature range, of which the first one is as good as any other. The devices with several + # cores have one page for each core, of which the last one is the main core. + pattern = re.escape(device.lower()).replace("x", r"\w") + r"(@m\d+)?" + chips = [c for c in _json(path / "manifest.json")["devices"] if re.fullmatch(pattern, c["chip_id"])] + if not chips: + return None + chip = max(chips, key=lambda c: (-ord(c["chip_id"][len(device) - 1]), c["chip_id"])) + rows = _json(path / chip["data"])["devices"][chip["chip_id"]]["rows"] + share = { + "version": 1, + "format": "modm-pinout-share", + "chipId": chip["chip_id"], + "state": share_state(rows, connectors, uses), + } + deflate = zlib.compressobj(9, zlib.DEFLATED, -15) + data = deflate.compress(json.dumps(share, separators=(",", ":")).encode()) + deflate.flush() + return f"{chip['page']}#share=v1d.{base64.urlsafe_b64encode(data).decode().rstrip('=')}" diff --git a/src/modm_data/html2pinout/signals.py b/src/modm_data/html2pinout/signals.py index 65e5a39..814f6f9 100644 --- a/src/modm_data/html2pinout/signals.py +++ b/src/modm_data/html2pinout/signals.py @@ -48,7 +48,10 @@ def device_for_board(board: str) -> tuple[str, str] | None: :return: The name of the mounted device and of its CubeMX description, or `None` if unknown. """ if not (match := re.fullmatch(r"NUCLEO-(\w+?)(?:-([PQ]))?", board)): - return None + # The names of the other boards do not contain the whole device name, but their board file does + mcu = dict(re.findall(r"^Mcu\.(Name|UserName)=(.*)$", _board_file(board), re.MULTILINE)) + exists = mcu and (_mcu_path() / f"{mcu['Name']}.xml").exists() + return (mcu["UserName"], mcu["Name"]) if exists else None smps = match.group(2) or "" # Some board names end in a revision digit, which is not part of the device name for part in ("STM32" + match.group(1), "STM32" + match.group(1)[:-1]): @@ -89,7 +92,13 @@ def device_signals(name: str) -> dict[str, dict[str, int | None]]: def _board_files(board: str) -> list[Path]: - return sorted((_cubemx_path() / "plugins/boardmanager/boards").glob(f"*_Nucleo_{board}_*_Board_AllConfig.ioc")) + return sorted((_cubemx_path() / "plugins/boardmanager/boards").glob(f"*_{board}_*_Board_AllConfig.ioc")) + + +def _board_file(board: str) -> str: + # The shortest name is the default configuration without TrustZone or multi-core variants + paths = _board_files(board) + return min(paths, key=lambda p: len(p.name)).read_text(errors="replace") if paths else "" def has_board_file(board: str) -> bool: @@ -103,11 +112,8 @@ def board_uses(board: str) -> dict[str, dict[str, str]]: :return: `gpio -> {use, label, part, signal}` for all GPIOs that are connected to something on the board itself, like the ST-LINK, an Ethernet PHY, a crystal, LEDs or buttons. """ - if not (paths := _board_files(board)): - return {} pins = {} - # The shortest name is the default configuration without TrustZone or multi-core variants - for line in min(paths, key=lambda p: len(p.name)).read_text(errors="replace").splitlines(): + for line in _board_file(board).splitlines(): if match := re.match(r"(P[A-Z]\d+)[^.=]*\.(GPIO_Label|Signal)=(.*)", line): pins.setdefault(match.group(1), {})[match.group(2)] = match.group(3).strip() diff --git a/test/html2pinout_test.py b/test/html2pinout_test.py index 7bf95c2..60bdfa0 100644 --- a/test/html2pinout_test.py +++ b/test/html2pinout_test.py @@ -3,7 +3,8 @@ import re -from modm_data.html2pinout import _groups, _signal, _field +from modm_data.html2pinout import _boards, _device_table_pinout, _groups, _signal, _field +from modm_data.html2pinout.share import share_state from modm_data.html2pinout.signals import _USES @@ -73,3 +74,73 @@ def test_transcribed(): assert sizes == {"UM2324": [38, 38], "UM2206": [38, 38], "UM2953": [38, 38], "UM2592": [12, 12], "UM2581": [2, 2]} assert _signal("5V-USB-CHG") == ("5V_USB_CHGR", "power") assert _signal("5V-STLINK") == ("5V_STLK", "power") + + +def test_boards(): + text = "NUCLEO-F429ZI 32F429IDISCOVERY STM32F4DISCOVERY STM32H573I-DK B-U585I-IOT02A STM32L552E-EV NUCLEO-G4XXRY" + assert _boards(text) == { + "NUCLEO-F429ZI", + "STM32F429I-DISCO", + "STM32F4DISCOVERY", + "STM32H573I-DK", + "B-U585I-IOT02A", + "STM32L552E-EV", + } + + +class _Table: + def __init__(self, hrows, rows): + self._hrows, self._rows = hrows, rows + self.rows, self.columns = len(rows), len(rows[0]) + + def cell(self, x, y): + text = self._rows[y][x] + return type("Cell", (), {"text": lambda self, **kw: text})() + + +def test_device_table(): + table = _Table( + 2, + [ + ["MCU pin", "MCU pin", "Board function", "Board function", "Board function", "Board function"], + ["Main function", "LQFP64", "LED", "Power supply", "P1", "P2"], + ["PA0- WKUP", "14", "-", "-", "15", "-"], + ["PC9", "40", "Green", "-", "-", "4"], + ["-", "-", "-", "GND", "1 33", "1"], + ], + ) + assert _device_table_pinout(table) == { + "P1": { + 15: {"signal": "PA0", "kind": "gpio", "pin": "PA0- WKUP"}, + 1: {"signal": "GND", "kind": "ground", "pin": "GND"}, + 33: {"signal": "GND", "kind": "ground", "pin": "GND"}, + }, + "P2": { + 4: {"signal": "PC9", "kind": "gpio", "pin": "PC9", "function": "LED: Green"}, + 1: {"signal": "GND", "kind": "ground", "pin": "GND"}, + }, + } + # Any other table has no connector columns + assert _device_table_pinout(_Table(1, [["Pin", "Main function"], ["1", "PA0"]])) == {} + + +def test_share_state(): + rows = [ + {"row_id": 0, "short_name": "PA0/WKUP", "functions": ["TIM2_CH1"]}, + {"row_id": 1, "short_name": "PA13", "functions": ["SYS_JTMS/SYS_SWDIO"]}, + {"row_id": 2, "short_name": "PG13", "functions": ["ETH_TXD0", "SPI6_SCK"]}, + {"row_id": 3, "short_name": "VDD", "functions": []}, + ] + connectors = { + "CN7": {"pins": {1: {"signal": "PA0", "kind": "gpio", "label": "A0"}, 2: {"signal": "VDD", "kind": "power"}}} + } + uses = { + "PA0": {"use": "button", "label": "B1", "signal": "GPXTI0"}, + "PA13": {"use": "debug", "label": "TMS", "signal": "SYS_JTMS-SWDIO"}, + "PG13": {"use": "ethernet", "label": "RMII_TXD0", "part": "LAN8742", "signal": "ETH_RMII_TXD0"}, + } + assert share_state(rows, connectors, uses) == { + "version": 1, + "selectedByRowId": {"1": ["SYS_JTMS/SYS_SWDIO"], "2": ["ETH_TXD0"]}, + "namesByRowId": {"0": "B1, CN7-1 (A0)", "1": "TMS", "2": "RMII_TXD0 [LAN8742]"}, + } diff --git a/tools/make/common.mk b/tools/make/common.mk index 2a2113d..7bd319f 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -68,7 +68,7 @@ build-homepage: --names ext/cache/svd2variants/feature-names-gemma.json --output docs/src/variants/ @curl -fsSL --create-dirs -o docs/src/variants/index.html \ https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/variants.html - @python3 -m modm_data.html2pinout --output docs/src/boards/nucleo.json + @python3 -m modm_data.html2pinout --output docs/src/boards/nucleo.json --pinout docs/src/pinout @curl -fsSL --create-dirs -o docs/src/boards/index.html \ https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/boards.html @(cd docs && mkdocs build)