diff --git a/.github/workflows/create-board-checkout-with-token.yaml b/.github/workflows/create-board-checkout-with-token.yaml index 29a1caf9..e5641c84 100644 --- a/.github/workflows/create-board-checkout-with-token.yaml +++ b/.github/workflows/create-board-checkout-with-token.yaml @@ -39,11 +39,10 @@ jobs: git submodule update --init fi - - name: Install kicad + - name: Install kicad 10 run: | - sudo add-apt-repository --yes ppa:kicad/kicad-8.0-releases + sudo add-apt-repository --yes ppa:kicad/kicad-10.0-releases sudo apt-get install kicad - pip install kicad-netlist-reader - name: Generate Gerber Files run: | diff --git a/.github/workflows/create-board.yaml b/.github/workflows/create-board.yaml index 006ca12a..0416c617 100644 --- a/.github/workflows/create-board.yaml +++ b/.github/workflows/create-board.yaml @@ -39,11 +39,10 @@ jobs: git submodule update --init fi - - name: Install kicad 8 + - name: Install kicad 10 run: | - sudo add-apt-repository --yes ppa:kicad/kicad-8.0-releases + sudo add-apt-repository --yes ppa:kicad/kicad-10.0-releases sudo apt-get install kicad - pip install kicad-netlist-reader - name: Generate Gerber Files run: | diff --git a/.github/workflows/test-check-all.yaml b/.github/workflows/test-check-all.yaml index 37b635a5..5ee88cf0 100644 --- a/.github/workflows/test-check-all.yaml +++ b/.github/workflows/test-check-all.yaml @@ -18,3 +18,30 @@ jobs: - name: Run Tests in Docker run: | bash step2-run-docker.sh + + kicad-export: + # Exercises kicad/bin/export.sh (the step board repos run via create-board.yaml) against KiCad 10 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install kicad 10 + run: | + sudo add-apt-repository --yes ppa:kicad/kicad-10.0-releases + sudo apt-get install kicad + + - name: Export test frame with kicad-cli + run: | + cd tests + bash ../kicad/bin/export.sh + + - name: Copy exported frame into project layout + run: | + python3 ./bin/copy_from_Kicad.py "frames:hellen" "tests" "../../gerber" "1test" "a" + + - name: Compare BOM and CPL with the expected output + run: | + diff tests/boards.EXAMPLE/hellen1test-a/frame/1test-BOM.csv tests/boards/hellen1test-a/frame/1test-BOM.csv + diff tests/boards.EXAMPLE/hellen1test-a/frame/1test-CPL.csv tests/boards/hellen1test-a/frame/1test-CPL.csv + ls -l tests/boards/hellen1test-a/frame diff --git a/CHANGELOG.md b/CHANGELOG.md index 63094045..2d7e1e69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes will be documented in this file. +## KiCad 10 toolset + - Frame export (kicad/bin/export.sh) and the create-board GitHub workflows now require KiCad 10 + - No more pcbnew Python scripting: zones are refilled by 'kicad-cli pcb export gerbers --check-zones', VRML by 'kicad-cli pcb export vrml'; the board file is no longer modified during export + - hellen-one-kicad-bom-plugin.py uses the kicad_netlist_reader shipped with KiCad and honours the native DNP attribute (in addition to MyComment=DNP) + - CI runs the KiCad export on the test frame (tests/) with KiCad 10 + ## knock-0.2 - now double-sided assembly and smaller dimentions diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..959df921 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,87 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this repo is + +Hellen One is a toolset for building custom rusEFI ECU PCBs by **merging gerbers of proven, versioned modules into a user-designed "frame" PCB**. The frame (drawn in KiCad) holds mainly the vehicle connector plus module footprints; this repo supplies the modules and the scripts that stitch everything into fab-ready output (merged gerbers, BOM/CPL for JLCPCB, schematic PDF, board renders, interactive HTML BOM). + +Board projects live in *separate* repos (e.g. `rusefi/hellen-example`, `rusefi/uaefi`) that include this repo as a `hellen-one` git submodule and call its reusable GitHub workflows. This repo is the shared engine; most day-to-day commits here are data edits (`global-replace.csv`, `board_id/`, module revisions). + +Submodules are required (`git submodule update --init --recursive`): `bin/gerbmerge`, `bin/pcb-tools`, `bin/python-combine-pdfs`, `bin/InteractiveHtmlBom`, `board_id/libfirmware`. + +## Commands + +The scripts hardcode `python3.8` (see `python_bin` in `bin/create_board.sh`, `bin/create_board_with_prefix.sh`, `bin/check_all.sh`). Docker is the supported way to run them locally. + +```bash +# Full test run in Docker (what CI does): builds image, runs run_tests.sh +bash step1-build-docker.sh +bash step2-run-docker.sh # optional arg: alternative script to run instead of run_tests.sh + +# Natively (needs python3.8, Xvfb, deps from bin/requirements.txt; check_all.sh is interactive) +bash run_tests.sh +``` + +`run_tests.sh` starts Xvfb on `:99`, runs `bin/check_all.sh` (dependency checker, prompts to install missing packages), then from `tests/` runs the two pipeline steps on the sample `hellen1test` frame. Expected output is in `tests/boards.EXAMPLE/hellen1test-a/`. There is no unit test suite; the pipeline running end to end is the test. + +Run the two steps individually (always from repo root, paths are relative to it): + +```bash +# Step 2: import KiCad export (tests/gerber/) into tests/boards/hellen1test-a/frame/ +python3 ./bin/copy_from_Kicad.py "frames:hellen" "tests" "../../gerber" "1test" "a" +# Step 3: merge modules and produce tests/boards/hellen1test-a/board/ +sh bin/create_board_with_prefix.sh "hellen" "tests" "1test" "a" "bom_replace_hellen1test-a.csv" "0,0" +``` + +Board repos drive the same thing via `bin/step1_build_hellen-one_docker.sh`, `bin/step2_copy_with_docker.sh`, `bin/step3_create_board_with_docker.sh`, which read `BOARD_PREFIX`, `BOARD_SUFFIX`, `BOARD_REVISION`, `BOARD_LAYERS`, `BOARD_PCB_OFFSET` from the board repo's `revision.txt`. + +Other commands: + +```bash +# Export gerbers/BOM/CPL/PDF/VRML from a KiCad frame (needs KiCad 10 kicad-cli; reads revision.txt in cwd) +bash kicad/bin/export.sh +# Same thing on the test frame (tests/revision.txt is committed): +cd tests && bash ../kicad/bin/export.sh + +# Allocate a new Board-ID: uncomment/add a line in board_id/test.sh, then +cd board_id && bash test.sh # writes generated/board_id_.csv, updates board_ids.csv and libfirmware/board_id/boards_id.h + +# Re-export a KiCad-designed module into modules/// (see kicad/modules/hellen1-*/copy_module_*.bat) +python ./bin/copy_from_Kicad.py "modules" "kicad/modules" "gerber" "wbo" "0.6" +``` + +## Pipeline architecture + +Three stages, each a separate entry point: + +1. **KiCad export** (`kicad/bin/export.sh`, requires KiCad 10, runs in GitHub Actions): pure `kicad-cli`, no `pcbnew` Python. Exports gerbers with `--check-zones` (in-memory zone refill, the board file is never modified), drill, positions CSV, schematic PDF and VRML (`pcb export vrml --user-origin` set from the board's `aux_axis_origin`). `kicad/hellen-one-kicad-bom-plugin.py` turns the XML netlist into a `Comment,Designator,Footprint,LCSC Part #` CSV using the `kicad_netlist_reader` shipped with KiCad (`/usr/share/kicad/plugins`, overridable with `KICAD_PLUGINS_DIR`); a `MyComment=DNP` field or the native DNP attribute blanks the LCSC number. Output goes to `gerber/` in the board repo. `kicad-cli` rewrites `*.kicad_prl`; `bin/gha-commit.sh` restores it so it is not committed. + +2. **Copy/normalize** (`bin/copy_from_Kicad.py`): copies KiCad gerbers into `boards/-/frame/` with Altium/JLC-style extensions (`.GTL/.GBL/.GTO/...`, inner `.G2/.G3` renamed to `.G1/.G2`, Edge.Cuts becomes both `.GKO` and `.GM15`). Rewrites the BOM: footprint names mapped through `kicad/footprints.csv`, library prefix stripped, and any component whose value matches `Module--` becomes `Module:/`. Builds the CPL from the positions file, applying per-footprint rotation corrections from `bin/jlc_kicad_tools/cpl_rotations_db.csv`. + +3. **Board creation** (`bin/process_board.py`, invoked by the `create_board*.sh` wrappers): the core. It deletes and recreates `boards//board/`, then: + - Scans the frame BOM for `Module:/` rows. For each, looks up the designator's position in the frame CPL, and pulls gerbers/BOM/CPL/schematic from `modules///`. Multiple instances of one module get `_2`, `_3` designator suffixes. Bottom-side modules get layers swapped, rotation inverted and a vertical flip. + - Writes a `gerbmerge` config + placement file and runs `bin/gerbmerge/gerbmerge` (coordinates converted mm to inch). Inner layers are only merged when `num_layers >= 4` and the module ships `.G1/.G2`. + - Runs `bin/process_BOM.py` (merges duplicate part numbers, applies the board's `bom_replace_*.csv`), then `bin/convert_BOM_mfr.py` (drops rows with no part number to produce the `-BOM-JLC.csv`). + - Merges schematic PDFs, renders top/bottom/outline PNGs via `bin/render_gerber.py` (pcb-tools + cairo), merges module VRML into a 3D model and renders components with ModernGL (`bin/render_vrml/`, needs the Xvfb display), composites the board image, and generates the iBOM with `bin/gen_iBOM.py` using footprints from `ibom-data/`. + - Zips the merged gerbers. + +## Data conventions + +- **Module layout**: `modules///.{GTL,GBL,GTS,GBS,GTO,GBO,GTP,GKO,GM15,DRL[,G1,G2,GBP]}`, `-BOM.csv`, `-CPL.csv`, `-schematic.pdf`, `-vrml.wrl`, plus `.kicad_mod`/`.kicad_sym` for use in frames. `.GM15` is the module border; `.GKO` is keepout. Module revisions are immutable: a change means a new revision folder and a `CHANGELOG.md` entry. Module-level designators `M` are stripped when merging. +- **Module sources**: only some modules were designed in KiCad; their sources are in `kicad/modules/hellen1-/`. Others came from Altium (`altium.shared/`). +- **BOM replacement CSVs** (`bom_replace_*.csv` in board repos, `global-replace.csv` and `board_id/generated/*.csv` here) are parsed by `process_BOM.py` with a mini preprocessor: `#include file.csv` (relative to the including file), `#define OLD_LCSC NEW_LCSC` (global part-number substitution), `#` comments, and rows `designator[,designator...],comment,footprint,lcsc` that move designators to a new part (a row with only a designator removes it, i.e. DNP). Non-ASCII characters are a hard error. `global-replace.csv` is the shared substitution list that board repos `#include`; most commits here edit it. +- **Board-ID**: each board revision encodes an ID in two resistors (R133/R134 on the mcu module), `id = R1_index*100 + R2_index` over `board_id/resistors.csv`. `board_ids.csv` is the registry; `gen_hellen_board_id.py` appends the next free ID and writes `generated/board_id_-.csv` for the board's bom_replace to include. The generated header goes into the `libfirmware` submodule. A BOM whose Board-ID resistors are still `?`/`board_id` fails processing until an ID file is included. +- **Naming**: project name is `` (default prefix `hellen`); board name adds `-`. Board repos are laid out as `boards//frame/` (input) and `boards//board/` (generated, committed by CI). + +## CI / reusable workflows + +`.github/workflows/test-check-all.yaml` runs two jobs on every push/PR: the Docker pipeline test, and a KiCad 10 job that runs `kicad/bin/export.sh` on `tests/` and diffs the resulting frame BOM/CPL against `tests/boards.EXAMPLE`. `create-board.yaml` and `create-board-checkout-with-token.yaml` are `workflow_call` workflows consumed by board repos: they install KiCad 10 from `ppa:kicad/kicad-10.0-releases`, run `kicad/bin/export.sh`, commit `gerber/`, then run the three Docker steps and commit `boards/`. `bin/gha-commit.sh` does the commit and sets `NOCOMMIT`/`YESCOMMIT`. `custom-board-update-hellen-one-reference.yaml` bumps the `hellen-one` submodule pointer in a board repo. + +## Gotchas + +- Module rotation in frames is only supported in multiples of 90 degrees. +- Board/module KiCad files must stay loadable by KiCad 10; `kicad-cli` also loads the `.kicad_pro` (net classes affect zone fills), so keep it next to the board. +- Scripts assume cwd is the repo root; `create_board.sh` refuses to run without args and is meant to be called from a user script like `create_hellen_board_example.sh`. +- `process_board.py` wipes the whole `board/` output directory on every run. +- `bin/check_all.sh` is interactive (uses `select`) and will prompt when a dependency is missing; in Docker all deps are preinstalled so it passes silently. diff --git a/bin/gha-commit.sh b/bin/gha-commit.sh index 798adcd4..7b55520d 100644 --- a/bin/gha-commit.sh +++ b/bin/gha-commit.sh @@ -4,6 +4,8 @@ git config --local user.name "GitHub create-board Action" echo "Status 1/3" git status git restore *.kicad_pro +# kicad-cli rewrites the project-local settings file; do not commit that noise +git restore *.kicad_prl echo "Status 2/3" git status git add gerber/* diff --git a/kicad/bin/export-vrml.py b/kicad/bin/export-vrml.py deleted file mode 100644 index 1e482b09..00000000 --- a/kicad/bin/export-vrml.py +++ /dev/null @@ -1,11 +0,0 @@ -import sys -from pcbnew import LoadBoard, EXPORTER_VRML - -b = LoadBoard(sys.argv[1]) -e = EXPORTER_VRML(b) - -# X and Y of Drill/Place origin are passed as parameters -x = float(sys.argv[2]) -y = float(sys.argv[3]) - -e.ExportVRML_File(b.GetProject(), None, sys.argv[4], 1.0, False, True, None, x, y) diff --git a/kicad/bin/export.sh b/kicad/bin/export.sh index d3edf418..19c7f00d 100755 --- a/kicad/bin/export.sh +++ b/kicad/bin/export.sh @@ -1,5 +1,11 @@ #!/bin/env bash +# +# Hellen-One: exports gerbers, drill, positions, BOM, schematic PDF and VRML from a KiCad frame project. +# Requires KiCad 10 (kicad-cli). Reads BOARD_PREFIX/BOARD_SUFFIX from revision.txt in the current folder. +# The board file is never modified: zone refill happens in-memory via 'kicad-cli pcb export gerbers --check-zones'. +# + # Get path of script so we can call python scripts DIR=$(dirname $0) @@ -17,38 +23,50 @@ then exit -1 fi +KICAD_CLI="${KICAD_CLI:-kicad-cli}" +PYTHON="${PYTHON:-python3}" + +KICAD_VERSION=$($KICAD_CLI version) +KICAD_MAJOR=${KICAD_VERSION%%.*} +echo "Using $KICAD_CLI $KICAD_VERSION" +if [ "$KICAD_MAJOR" -lt 10 ] +then + echo "KiCad 10 or later is required, found $KICAD_VERSION" + exit -1 +fi + OUT_FOLDER=gerber mkdir -p $OUT_FOLDER -# Copy to backup so we can modify before exporting -cp "$IN.kicad_pcb" "$IN.kicad_pcb.bak" - SCHEMATIC_FILE=$IN.kicad_sch PCB_FILE=$IN.kicad_pcb NET_FILE=$OUT_FOLDER/$IN.net if [ ! -f $SCHEMATIC_FILE ] then - echo "[$SCHEMATIC_FILE] schematic does not exist make sure at least KiCAD 6.0" + echo "[$SCHEMATIC_FILE] schematic does not exist" + exit -1 +fi +if [ ! -f $PCB_FILE ] +then + echo "[$PCB_FILE] board does not exist" exit -1 fi + echo Export PDF from [$SCHEMATIC_FILE] schematic -kicad-cli sch export pdf "$SCHEMATIC_FILE" --no-background-color -o "gerber/$IN.pdf" +$KICAD_CLI sch export pdf "$SCHEMATIC_FILE" --no-background-color -o "$OUT_FOLDER/$IN.pdf" echo Export netlist from [$SCHEMATIC_FILE] schematic into [$NET_FILE] -kicad-cli sch export netlist "$SCHEMATIC_FILE" --format kicadxml -o "$NET_FILE" +$KICAD_CLI sch export netlist "$SCHEMATIC_FILE" --format kicadxml -o "$NET_FILE" echo Run BOM plugin script on [$NET_FILE] -python "$DIR/../hellen-one-kicad-bom-plugin.py" "$OUT_FOLDER/$IN.net" "$OUT_FOLDER/$IN.csv" - -echo Fill zones -python "$DIR/fill-zones.py" "$PCB_FILE" +$PYTHON "$DIR/../hellen-one-kicad-bom-plugin.py" "$NET_FILE" "$OUT_FOLDER/$IN.csv" -echo Export Gerbers, drill file, and positions file -kicad-cli pcb export gerbers --disable-aperture-macros -l "F.Cu,B.Cu,F.Paste,B.Paste,F.SilkS,B.SilkS,F.Mask,B.Mask,Edge.Cuts,In2.Cu,In1.Cu" --no-x2 --use-drill-file-origin "$PCB_FILE" -o gerber/ +echo Export Gerbers with zones refilled in-memory +$KICAD_CLI pcb export gerbers --disable-aperture-macros -l "F.Cu,B.Cu,F.Paste,B.Paste,F.SilkS,B.SilkS,F.Mask,B.Mask,Edge.Cuts,In2.Cu,In1.Cu" --no-x2 --use-drill-file-origin --check-zones "$PCB_FILE" -o $OUT_FOLDER/ echo Export drill file -kicad-cli pcb export drill --map-format ps --drill-origin plot --excellon-zeros-format suppressleading -u "in" "$PCB_FILE" -o gerber/ +$KICAD_CLI pcb export drill --map-format ps --drill-origin plot --excellon-zeros-format suppressleading -u "in" "$PCB_FILE" -o $OUT_FOLDER/ echo Export positions file -kicad-cli pcb export pos --format csv --units mm --use-drill-file-origin --bottom-negate-x "$PCB_FILE" -o "gerber/$IN-all-pos.csv" +$KICAD_CLI pcb export pos --format csv --units mm --use-drill-file-origin --bottom-negate-x "$PCB_FILE" -o "$OUT_FOLDER/$IN-all-pos.csv" echo Getting Drill/Place origin from PCB X=$(grep "aux_axis_origin" "$PCB_FILE" | tr ' ' ' ' | tr -s ' ' | cut -d ' ' -f 3) @@ -57,8 +75,7 @@ if [ ! "$Y" ]; then echo "aux_axis_origin is missing in the PCB file" exit -1 fi -echo Export VRML using $X $Y -python "$DIR/export-vrml.py" "$PCB_FILE" "$X" "$Y" "gerber/$IN.wrl" +echo Export VRML using origin $X $Y mm +$KICAD_CLI pcb export vrml --units mm --user-origin "${X}x${Y}mm" --force "$PCB_FILE" -o "$OUT_FOLDER/$IN.wrl" -echo Restore PCB backup -cp "$IN.kicad_pcb.bak" "$PCB_FILE" +echo Export done diff --git a/kicad/bin/fill-zones.py b/kicad/bin/fill-zones.py deleted file mode 100644 index ef694d59..00000000 --- a/kicad/bin/fill-zones.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys -from pcbnew import LoadBoard, ZONE_FILLER, SaveBoard - -b = LoadBoard(sys.argv[1]) -bz = b.Zones() -zf = ZONE_FILLER(b).Fill(bz) - -SaveBoard(sys.argv[1], b) diff --git a/kicad/hellen-one-kicad-bom-plugin.py b/kicad/hellen-one-kicad-bom-plugin.py index 3b96917a..1d289133 100644 --- a/kicad/hellen-one-kicad-bom-plugin.py +++ b/kicad/hellen-one-kicad-bom-plugin.py @@ -22,8 +22,16 @@ import sys import os -# [andreika]: add kicad plugins path -sys.path.append(os.path.dirname(sys.executable) + "/scripting/plugins") +# Locate kicad_netlist_reader.py shipped with KiCad (Linux .deb, macOS bundle, Windows installer). +# KICAD_PLUGINS_DIR can point to it explicitly; 'pip install kicad-netlist-reader' also works. +for plugins_dir in [ + os.environ.get("KICAD_PLUGINS_DIR", ""), + "/usr/share/kicad/plugins", + "/usr/local/share/kicad/plugins", + "/Applications/KiCad/KiCad.app/Contents/SharedSupport/plugins", + os.path.dirname(sys.executable) + "/scripting/plugins"]: + if plugins_dir and os.path.isdir(plugins_dir) and plugins_dir not in sys.path: + sys.path.append(plugins_dir) # Import the KiCad python helper module import kicad_netlist_reader @@ -59,7 +67,8 @@ def writerow( acsvwriter, columns ): # Output all of the component information (One component per row) for c in components: lcsc = c.getField("LCSC") - if (c.getField("MyComment") == "DNP"): + # Do-Not-Populate: either the legacy MyComment=DNP field or the native KiCad DNP attribute + if (c.getField("MyComment") == "DNP" or (hasattr(c, "getDNP") and c.getDNP())): lcsc = "" writerow( out, [c.getValue(), c.getRef(), c.getFootprint(), lcsc]) diff --git a/kicad/readme.md b/kicad/readme.md index 97cd17a6..f9348526 100644 --- a/kicad/readme.md +++ b/kicad/readme.md @@ -5,3 +5,15 @@ If you are designing a new ECU frame see https://github.com/andreika-git/hellen- doThis folder contains module _source_ files, not reusable resulting frames like [here](../modules). See https://github.com/andreika-git/hellen-one/wiki/scripts-howto#how-to-create-a-module + +## Toolchain + +The export scripts in `bin/` target **KiCad 10** (`kicad-cli`). `export.sh` runs from a board repository root +(reads `revision.txt`) and writes `gerber/`: gerbers with zones refilled in-memory, drill, position CSV, +netlist + BOM CSV (via `hellen-one-kicad-bom-plugin.py`), schematic PDF and VRML. The board file itself is not modified. + +Local run for the test frame: + +``` +cd tests && bash ../kicad/bin/export.sh +``` diff --git a/tests/revision.txt b/tests/revision.txt new file mode 100644 index 00000000..eaedf32e --- /dev/null +++ b/tests/revision.txt @@ -0,0 +1,4 @@ +BOARD_PREFIX=hellen +BOARD_SUFFIX=1test +BOARD_REVISION=a +BOARD_LAYERS=2