diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4b8af23..cd4e18b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -92,4 +92,4 @@ nav: - Pinouts: pinout/index.html - SVD Explorer: svd/index.html - Peripheral Variants: variants/index.html - - NUCLEO Pinouts: boards/index.html + - Board Pinouts: boards/index.html diff --git a/src/modm_data/html2pinout/__init__.py b/src/modm_data/html2pinout/__init__.py index 258b70e..f0dd8b0 100644 --- a/src/modm_data/html2pinout/__init__.py +++ b/src/modm_data/html2pinout/__init__.py @@ -4,8 +4,9 @@ """ # HTML to Board Pinout Pipeline -Extracts the connector pinouts (ST morpho, ARDUINO, Zio) of the NUCLEO boards -from the connector tables of their user manuals. +Extracts the connector pinouts (ST morpho, ARDUINO, Zio, STMod+, Pmod, extension) +of the NUCLEO boards and Discovery kits from the connector tables of their user +manuals. """ import re @@ -18,15 +19,22 @@ LOGGER = logging.getLogger(__name__) -_TABLES = r"morpho|arduino|zio" -_NUMBER = re.compile(r"^(zio|connector)?pin(number|no\.?|nbr)?$") +_TABLES = r"morpho|arduino|zio|stmod|pmod|mikrobus|extension" +# The daughterboard connectors of the evaluation boards are unique to each board +_SKIPPED = r"daughter ?board|shield" +_NUMBER = re.compile(r"^(zio|connector)?pin(number|no\.?|nbr)?(\((cn|p)\d+\))?$") _CONNECTOR = re.compile(r"^(connector|cn)$") _MCU = re.compile(r"(stm32|mcu|stlink).*(pin?|port|name)$|^port$") _GPIO = re.compile(r"(? str: def _boards(html: str) -> set[str]: - return set(_BOARD.findall(html)) + # The Discovery kits also have a marketing name: 32F429IDISCOVERY is the STM32F429I-DISCO + return set(_BOARD.findall(re.sub(r"\b32([A-Z]\d\w+?)DISCOVERY\b", r"STM32\1-DISCO", html))) def _document_boards(chapters: list) -> set[str]: @@ -133,17 +142,19 @@ def _table_pinout(table, kind: str) -> dict[str, dict]: rows = range(table._hrows, table.rows) if not (groups := _groups(header)): # Some tables only name the connectors in the header, then the pin numbers are found by their content - digits = [sum(_text(table.cell(x, y)).isdigit() for y in rows) for x in range(table.columns)] + cells = [[_text(table.cell(x, y)) for y in rows] for x in range(table.columns)] + # Repeated numbers are the values of a configuration table instead + digits = [sum(c.isdigit() for c in column) * (len(set(column)) > len(column) / 2) for column in cells] hy, header = table._hrows, ["Pin" if d > len(rows) * 0.8 else "Pin name" for d in digits] groups = _groups(header) - captions = set(_CN.findall(table.caption())) + captions = set(_NAME.findall(table.caption())) pinout = defaultdict(dict) unnamed = 1 for number, columns in groups.items(): # The connector is named in the header above, in its own column or in the caption - above = " ".join(_text(table.cell(number, y)) for y in range(hy)) - connector = next(iter(_CN.findall(above)), next(iter(captions), "") if len(captions) == 1 else "") + above = " ".join(_text(table.cell(number, y)) for y in range(hy + 1)) + connector = next(iter(_NAME.findall(above)), next(iter(captions), "") if len(captions) == 1 else "") connector_columns = [ c for c in columns @@ -185,6 +196,49 @@ def _table_pinout(table, kind: str) -> dict[str, dict]: return {c: pins for c, pins in pinout.items() if pins} +def _device_table_pinout(table) -> dict[str, dict]: + """ + The older Discovery kits have one table of all device pins instead of one per connector: one row per device + pin, with one column per board function and per extension connector (`P1`, `P2`), which contains the + connector pin numbers. + """ + if table._hrows < 2: + return {} + header = [_text(table.cell(x, table._hrows - 1)) for x in range(table.columns)] + keys = [_key(h) for h in header] + if "mainfunction" not in keys: + return {} + main = keys.index("mainfunction") + power = keys.index("powersupply") if "powersupply" in keys else main + connectors = [x for x, h in enumerate(header) if re.fullmatch(r"P\d", h)] + functions = [ + x + for x in range(table.columns) + if _key(_text(table.cell(x, 0))).startswith("boardfunction") + and x not in connectors + and keys[x] not in {"freei/o", "powersupply"} + and not _CN.fullmatch(header[x]) + ] + pinout = defaultdict(dict) + for y in range(table._hrows, table.rows): + # The supplies are not device pins + name = _text(table.cell(main, y)) + name = _text(table.cell(power, y)) if name in {"", "-"} else name + signal, kind = _signal(name) + texts = [(header[x], _text(table.cell(x, y))) for x in functions] + function = " / ".join(f"{h}: {text}" for h, text in texts if text not in {"", "-"}) + for x in connectors: + for number in re.findall(r"\d+", _text(table.cell(x, y))): + pinout[header[x]][int(number)] = { + "signal": signal, + "kind": kind, + "pin": name, + **({"function": function} if function else {}), + } + # P1 is the left connector + return dict(sorted(pinout.items())) + + def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, dict]]: """ :return: `board -> connector -> {type, side, pins: number -> {signal, kind, ...}}` for all NUCLEO boards in @@ -201,7 +255,12 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di kind = re.search(_TABLES, table.caption(), re.IGNORECASE) or re.search( _TABLES, table.heading(), re.IGNORECASE ) - if not kind or not (pinout := _table_pinout(table, kind.group(0).lower())): + if not kind or re.search(_SKIPPED, table.caption() + table.heading(), re.IGNORECASE): + continue + kind = kind.group(0).lower() + if pinout := _device_table_pinout(table): + kind = "extension" + elif not (pinout := _table_pinout(table, kind)): continue # Some manuals describe several boards with one table for each for board in _boards(table.caption()) or boards: @@ -211,7 +270,7 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di if sum(n in known and known[n]["signal"] != p["signal"] for n, p in pins.items()) > len(pins) / 2: LOGGER.warning(f"{document.fullname}: {board} {connector} is redefined in '{table.caption()}'") connector += "-2" - current = pinouts[board].setdefault(connector, {"type": kind.group(0).lower(), "pins": {}})["pins"] + current = pinouts[board].setdefault(connector, {"type": kind, "pins": {}})["pins"] for number, pin in pins.items(): if number in current and current[number]["signal"] != pin["signal"]: LOGGER.warning( @@ -231,6 +290,10 @@ def board_pinouts_from_user_manual(document: Document) -> dict[str, dict[str, di pins.setdefault(number, {"signal": signal, "kind": kind, "label": name, "remark": f"From {source}"}) for connectors in pinouts.values(): + # A table without a connector name describes the named connector of the same type once more + named = {c["type"] for name, c in connectors.items() if not name.startswith(c["type"])} + for name in [n for n, c in connectors.items() if n.startswith(c["type"]) and c["type"] in named]: + del connectors[name] for connector in connectors.values(): connector["pins"] = dict(sorted(connector["pins"].items())) # The manuals list the left morpho connector first diff --git a/src/modm_data/html2pinout/__main__.py b/src/modm_data/html2pinout/__main__.py index 5abf93e..34fb9dd 100644 --- a/src/modm_data/html2pinout/__main__.py +++ b/src/modm_data/html2pinout/__main__.py @@ -13,6 +13,7 @@ inherit_bridge_defaults, ) from modm_data.html2pinout.signals import board_uses, device_for_board, device_signals, has_board_file +from modm_data.html2pinout.share import pinout_link from modm_data.utils import ext_path @@ -25,6 +26,7 @@ def main(): parser.add_argument("--document", type=Path, help="A single user manual, otherwise all are converted.") parser.add_argument("--input", type=Path, default=ext_path("stmicro/html-archive")) parser.add_argument("--output", type=Path, default=ext_path("stmicro/pinout/nucleo.json")) + parser.add_argument("--pinout", type=Path, help="The output folder of modm_pinout to link the boards to.") args = parser.parse_args() logging.basicConfig(level=logging.WARNING, format="%(message)s") @@ -39,14 +41,17 @@ def main(): latest[name] = path paths = sorted(latest.values()) - pinouts = {} + pinouts, scores = {}, {} for path in paths: document = Document(path.absolute()) - for board, connectors in board_pinouts_from_user_manual(document).items(): - pins = sum(len(c["pins"]) for c in connectors.values()) - # Several manuals may name a board, the one that describes the most of it is its own - if board in pinouts and pins <= sum(len(c["pins"]) for c in pinouts[board]["connectors"].values()): + boards = board_pinouts_from_user_manual(document) + for board, connectors in boards.items(): + # Several manuals may name a board, the one that describes the most of it is its own. The manuals + # of the Discovery kits however also name their predecessors, which have their own smaller manual. + score = (board.startswith("NUCLEO") or -len(boards), sum(len(c["pins"]) for c in connectors.values())) + if board in pinouts and score <= scores[board]: continue + scores[board] = score pinouts[board] = {"board": board, "document": document.fullname, "connectors": connectors} bridges = board_bridges_from_user_manual(document) # Older revisions of the manual may still know the defaults that this one lost @@ -65,6 +70,9 @@ def main(): # NUCLEO-WL55JC1. A real board like NUCLEO-L452RE next to NUCLEO-L452RE-P has its own description. if any(other != board and other.startswith(board) for other in pinouts) and not has_board_file(board): continue + # The Discovery kits have marketing names and placeholders next to their real name + if not board.startswith("NUCLEO") and not has_board_file(board): + continue # Only the signals of the GPIOs on the connectors are of interest gpios = {p["signal"] for c in data["connectors"].values() for p in c["pins"].values() if p["kind"] == "gpio"} device = device_for_board(board) @@ -74,9 +82,14 @@ def main(): # Only the bridges that mention a GPIO on the connectors bridges = [dict(b, gpios=[g for g in b["gpios"] if g in gpios]) for b in data.pop("bridges", [])] data["bridges"] = [b for b in bridges if b["gpios"]] - data["uses"] = {gpio: use for gpio, use in board_uses(board).items() if gpio in gpios} + uses = board_uses(board) + data["uses"] = {gpio: use for gpio, use in uses.items() if gpio in gpios} + if device and args.pinout and (link := pinout_link(args.pinout, device[0], data["connectors"], uses)): + data["pinout"] = link boards.append(data) - sizes = ", ".join(f"{c}={len(p['pins'])}" for c, p in data["connectors"].items() if p["type"] == "morpho") + sizes = ", ".join( + f"{c}={len(p['pins'])}" for c, p in data["connectors"].items() if p["type"] in {"morpho", "extension"} + ) print(f"{board:20} {data['document']:12} {data['device'] or '-':16} {sizes}") if not boards: diff --git a/src/modm_data/html2pinout/share.py b/src/modm_data/html2pinout/share.py new file mode 100644 index 0000000..a9da372 --- /dev/null +++ b/src/modm_data/html2pinout/share.py @@ -0,0 +1,86 @@ +# Copyright 2026, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +Links a board to the page of its device in the pinout generator of `modm_pinout`, +so that the generator starts with what the board already uses the pins for. +""" + +import re +import json +import zlib +import base64 +from functools import cache +from pathlib import Path + + +@cache +def _json(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _function(signal: str, functions: list[str]) -> str | None: + # CubeMX calls it SYS_JTMS-SWDIO and ETH_RMII_TXD0, the generator SYS_JTMS/SYS_SWDIO and ETH_TXD0 + names = {signal, signal.split("-")[0], re.sub(r"_R?MII", "", signal), re.sub(r"^S_", "", signal)} + return next((f for f in functions if names & set(f.split("/"))), None) + + +def share_state(rows: list[dict], connectors: dict, uses: dict) -> dict: + """ + :param rows: The pins of the device in the generator: `{row_id, short_name, functions}`. + :param connectors: The connectors of the board. + :param uses: What the board itself uses the GPIOs for. + :return: The state of the generator with the functions used by the board selected, and everything else that + is known about a pin as its comment. + """ + comments = {} + for name, connector in connectors.items(): + for number, pin in connector["pins"].items(): + if pin["kind"] == "gpio": + # The ARDUINO pin names are better known than their position + label = pin.get("label", "") + label = f" ({label})" if re.fullmatch(r"[AD]\d+", label) else "" + comments.setdefault(pin["signal"], []).append(f"{name}-{number}{label}") + + selected, names = {}, {} + for row in rows: + # The generator names the pins PA0/WKUP or PC14/OSC32_IN + gpio = row["short_name"].split("/")[0] + use = uses.get(gpio, {}) + function = _function(use.get("signal", ""), row["functions"]) + if function: + selected[str(row["row_id"])] = [function] + label = " ".join(filter(None, [use.get("label"), use.get("part") and f"[{use['part']}]"])) + # A function that the generator does not know is still worth knowing + label = label or ("" if function else use.get("signal", "")) + if comment := ", ".join(filter(None, [label, *comments.get(gpio, [])])): + names[str(row["row_id"])] = comment + return {"version": 1, "selectedByRowId": selected, "namesByRowId": names} + + +def pinout_link(path: Path, device: str, connectors: dict, uses: dict) -> str | None: + """ + :param path: The output folder of `modm_pinout`. + :param device: The name of the device on the board like `STM32F429ZITx`. + :return: The page of the device relative to the folder with the state of the board as shared URL, or `None` + if the generator does not know the device. + """ + if not (path / "manifest.json").exists(): + return None + # The x is the temperature range, of which the first one is as good as any other. The devices with several + # cores have one page for each core, of which the last one is the main core. + pattern = re.escape(device.lower()).replace("x", r"\w") + r"(@m\d+)?" + chips = [c for c in _json(path / "manifest.json")["devices"] if re.fullmatch(pattern, c["chip_id"])] + if not chips: + return None + chip = max(chips, key=lambda c: (-ord(c["chip_id"][len(device) - 1]), c["chip_id"])) + rows = _json(path / chip["data"])["devices"][chip["chip_id"]]["rows"] + share = { + "version": 1, + "format": "modm-pinout-share", + "chipId": chip["chip_id"], + "state": share_state(rows, connectors, uses), + } + deflate = zlib.compressobj(9, zlib.DEFLATED, -15) + data = deflate.compress(json.dumps(share, separators=(",", ":")).encode()) + deflate.flush() + return f"{chip['page']}#share=v1d.{base64.urlsafe_b64encode(data).decode().rstrip('=')}" diff --git a/src/modm_data/html2pinout/signals.py b/src/modm_data/html2pinout/signals.py index 65e5a39..814f6f9 100644 --- a/src/modm_data/html2pinout/signals.py +++ b/src/modm_data/html2pinout/signals.py @@ -48,7 +48,10 @@ def device_for_board(board: str) -> tuple[str, str] | None: :return: The name of the mounted device and of its CubeMX description, or `None` if unknown. """ if not (match := re.fullmatch(r"NUCLEO-(\w+?)(?:-([PQ]))?", board)): - return None + # The names of the other boards do not contain the whole device name, but their board file does + mcu = dict(re.findall(r"^Mcu\.(Name|UserName)=(.*)$", _board_file(board), re.MULTILINE)) + exists = mcu and (_mcu_path() / f"{mcu['Name']}.xml").exists() + return (mcu["UserName"], mcu["Name"]) if exists else None smps = match.group(2) or "" # Some board names end in a revision digit, which is not part of the device name for part in ("STM32" + match.group(1), "STM32" + match.group(1)[:-1]): @@ -89,7 +92,13 @@ def device_signals(name: str) -> dict[str, dict[str, int | None]]: def _board_files(board: str) -> list[Path]: - return sorted((_cubemx_path() / "plugins/boardmanager/boards").glob(f"*_Nucleo_{board}_*_Board_AllConfig.ioc")) + return sorted((_cubemx_path() / "plugins/boardmanager/boards").glob(f"*_{board}_*_Board_AllConfig.ioc")) + + +def _board_file(board: str) -> str: + # The shortest name is the default configuration without TrustZone or multi-core variants + paths = _board_files(board) + return min(paths, key=lambda p: len(p.name)).read_text(errors="replace") if paths else "" def has_board_file(board: str) -> bool: @@ -103,11 +112,8 @@ def board_uses(board: str) -> dict[str, dict[str, str]]: :return: `gpio -> {use, label, part, signal}` for all GPIOs that are connected to something on the board itself, like the ST-LINK, an Ethernet PHY, a crystal, LEDs or buttons. """ - if not (paths := _board_files(board)): - return {} pins = {} - # The shortest name is the default configuration without TrustZone or multi-core variants - for line in min(paths, key=lambda p: len(p.name)).read_text(errors="replace").splitlines(): + for line in _board_file(board).splitlines(): if match := re.match(r"(P[A-Z]\d+)[^.=]*\.(GPIO_Label|Signal)=(.*)", line): pins.setdefault(match.group(1), {})[match.group(2)] = match.group(3).strip() diff --git a/test/html2pinout_test.py b/test/html2pinout_test.py index 7bf95c2..60bdfa0 100644 --- a/test/html2pinout_test.py +++ b/test/html2pinout_test.py @@ -3,7 +3,8 @@ import re -from modm_data.html2pinout import _groups, _signal, _field +from modm_data.html2pinout import _boards, _device_table_pinout, _groups, _signal, _field +from modm_data.html2pinout.share import share_state from modm_data.html2pinout.signals import _USES @@ -73,3 +74,73 @@ def test_transcribed(): assert sizes == {"UM2324": [38, 38], "UM2206": [38, 38], "UM2953": [38, 38], "UM2592": [12, 12], "UM2581": [2, 2]} assert _signal("5V-USB-CHG") == ("5V_USB_CHGR", "power") assert _signal("5V-STLINK") == ("5V_STLK", "power") + + +def test_boards(): + text = "NUCLEO-F429ZI 32F429IDISCOVERY STM32F4DISCOVERY STM32H573I-DK B-U585I-IOT02A STM32L552E-EV NUCLEO-G4XXRY" + assert _boards(text) == { + "NUCLEO-F429ZI", + "STM32F429I-DISCO", + "STM32F4DISCOVERY", + "STM32H573I-DK", + "B-U585I-IOT02A", + "STM32L552E-EV", + } + + +class _Table: + def __init__(self, hrows, rows): + self._hrows, self._rows = hrows, rows + self.rows, self.columns = len(rows), len(rows[0]) + + def cell(self, x, y): + text = self._rows[y][x] + return type("Cell", (), {"text": lambda self, **kw: text})() + + +def test_device_table(): + table = _Table( + 2, + [ + ["MCU pin", "MCU pin", "Board function", "Board function", "Board function", "Board function"], + ["Main function", "LQFP64", "LED", "Power supply", "P1", "P2"], + ["PA0- WKUP", "14", "-", "-", "15", "-"], + ["PC9", "40", "Green", "-", "-", "4"], + ["-", "-", "-", "GND", "1 33", "1"], + ], + ) + assert _device_table_pinout(table) == { + "P1": { + 15: {"signal": "PA0", "kind": "gpio", "pin": "PA0- WKUP"}, + 1: {"signal": "GND", "kind": "ground", "pin": "GND"}, + 33: {"signal": "GND", "kind": "ground", "pin": "GND"}, + }, + "P2": { + 4: {"signal": "PC9", "kind": "gpio", "pin": "PC9", "function": "LED: Green"}, + 1: {"signal": "GND", "kind": "ground", "pin": "GND"}, + }, + } + # Any other table has no connector columns + assert _device_table_pinout(_Table(1, [["Pin", "Main function"], ["1", "PA0"]])) == {} + + +def test_share_state(): + rows = [ + {"row_id": 0, "short_name": "PA0/WKUP", "functions": ["TIM2_CH1"]}, + {"row_id": 1, "short_name": "PA13", "functions": ["SYS_JTMS/SYS_SWDIO"]}, + {"row_id": 2, "short_name": "PG13", "functions": ["ETH_TXD0", "SPI6_SCK"]}, + {"row_id": 3, "short_name": "VDD", "functions": []}, + ] + connectors = { + "CN7": {"pins": {1: {"signal": "PA0", "kind": "gpio", "label": "A0"}, 2: {"signal": "VDD", "kind": "power"}}} + } + uses = { + "PA0": {"use": "button", "label": "B1", "signal": "GPXTI0"}, + "PA13": {"use": "debug", "label": "TMS", "signal": "SYS_JTMS-SWDIO"}, + "PG13": {"use": "ethernet", "label": "RMII_TXD0", "part": "LAN8742", "signal": "ETH_RMII_TXD0"}, + } + assert share_state(rows, connectors, uses) == { + "version": 1, + "selectedByRowId": {"1": ["SYS_JTMS/SYS_SWDIO"], "2": ["ETH_TXD0"]}, + "namesByRowId": {"0": "B1, CN7-1 (A0)", "1": "TMS", "2": "RMII_TXD0 [LAN8742]"}, + } diff --git a/tools/make/common.mk b/tools/make/common.mk index 2a2113d..7bd319f 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -68,7 +68,7 @@ build-homepage: --names ext/cache/svd2variants/feature-names-gemma.json --output docs/src/variants/ @curl -fsSL --create-dirs -o docs/src/variants/index.html \ https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/variants.html - @python3 -m modm_data.html2pinout --output docs/src/boards/nucleo.json + @python3 -m modm_data.html2pinout --output docs/src/boards/nucleo.json --pinout docs/src/pinout @curl -fsSL --create-dirs -o docs/src/boards/index.html \ https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/boards.html @(cd docs && mkdocs build)