diff --git a/.github/workflows/build-numpy.yml b/.github/workflows/build-numpy.yml index 3d7318c..663afef 100644 --- a/.github/workflows/build-numpy.yml +++ b/.github/workflows/build-numpy.yml @@ -24,6 +24,10 @@ env: UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ UV_INDEX_STRATEGY: unsafe-best-match UV_ONLY_BINARY: ':all:' + # Pinned explicitly (rather than left to cibuildwheel's default resolution) + # so the gpl_sources job below can pull the exact same image the wheels + # were built in. + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 jobs: build_wheels: @@ -79,6 +83,7 @@ jobs: CCACHE_BASEDIR CCACHE_COMPILERCHECK CIBW_CONTAINER_ENGINE: "docker; create_args: --volume ${{ env.CCACHE_DIR }}:/root/.ccache" + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} - name: Save compilation cache if: always() && github.ref == 'refs/heads/main' @@ -93,9 +98,26 @@ jobs: path: ./wheelhouse/*.whl if-no-files-found: error + gpl_sources: + name: Collect GPL sources (gcc) for numpy ${{ inputs.version || '2.5.1' }} + runs-on: ubuntu-24.04-riscv + steps: + - name: Collect gcc source RPM from manylinux_riscv64 + uses: riseproject-dev/python-wheels/actions/collect-gpl-sources@main + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: numpy-${{ env.NUMPY_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + publish: name: Publish numpy ${{ inputs.version || '2.5.1' }} to GitLab - needs: build_wheels + needs: [build_wheels, gpl_sources] # Only publish when the workflow was triggered from main with a specific # version. Manual trigger is the only entry point, so checking the ref is # enough to gate uploads. @@ -114,3 +136,6 @@ jobs: gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} gh-token: ${{ secrets.GITHUB_TOKEN }} + gpl-sources-artifact: numpy-${{ env.NUMPY_VERSION }}-gpl-sources + gpl-sources-release-tag: numpy-v${{ env.NUMPY_VERSION }} + gpl-sources-description: gcc diff --git a/actions/collect-gpl-sources/action.yml b/actions/collect-gpl-sources/action.yml new file mode 100644 index 0000000..87a8e08 --- /dev/null +++ b/actions/collect-gpl-sources/action.yml @@ -0,0 +1,81 @@ +name: 'Collect GPL Source RPMs from a manylinux Container' +description: > + Pulls the manylinux/musllinux container image used to build a package's + riscv64 wheels and downloads the source RPM(s) for one or more of its + installed packages (e.g. gcc), satisfying GPL source-distribution + requirements for the toolchain baked into the container. The collected + .src.rpm files are bundled into a single tar archive. + +inputs: + + # ── Required ──────────────────────────────────────────────────────────────── + + image: + description: > + Container image reference to pull GPL sources from, e.g. + quay.io/pypa/manylinux_2_39_riscv64. Should match the image actually + used to build the wheels (e.g. via CIBW_MANYLINUX_RISCV64_IMAGE) so the + published sources correspond to the toolchain that produced them. + required: true + + # ── Optional ──────────────────────────────────────────────────────────────── + + packages: + description: Space-separated list of installed RPM package names to fetch source RPMs for. + required: false + default: 'gcc' + + output: + description: Path of the tar archive to produce. + required: false + default: 'gpl-sources.tar' + +runs: + using: 'composite' + steps: + + - name: Pull container image + shell: bash + run: docker pull "${{ inputs.image }}" + + - name: Download source RPMs from the container's package repos + shell: bash + run: | + set -euxo pipefail + + workdir="$(mktemp -d)" + + # Package name and config-manager syntax for enabling the *-source + # repos differ between dnf4 (dnf-plugins-core, `--set-enabled`) and + # dnf5 (dnf5-plugins, `setopt .enabled=1`). Rocky/RHEL 10 based + # manylinux images ship dnf5, but both forms are attempted so this + # keeps working if the base image changes. + docker run --rm -v "${workdir}:/out" "${{ inputs.image }}" bash -c ' + set -euxo pipefail + dnf -y install dnf5-plugins || dnf -y install dnf-plugins-core + for repo in baseos-source appstream-source crb-source; do + dnf -y config-manager setopt "${repo}.enabled=1" \ + || dnf -y config-manager --set-enabled "$repo" \ + || true + done + dnf -y download --source --destdir /out ${{ inputs.packages }} + ' + + found=$(find "${workdir}" -maxdepth 1 -name "*.src.rpm" -printf '%f\n') + if [[ -z "${found}" ]]; then + echo "::error::No .src.rpm files were collected from ${{ inputs.image }} for packages: ${{ inputs.packages }}" + exit 1 + fi + printf '%s\n' "${found}" + + tar -cf "${{ inputs.output }}" -C "${workdir}" . + rm -rf "${workdir}" + + - name: Print summary + shell: bash + run: | + echo "### GPL Sources" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Collected from \`${{ inputs.image }}\`:" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + tar -tf "${{ inputs.output }}" | sed 's/^/- `/;s/$/`/' >> "$GITHUB_STEP_SUMMARY" diff --git a/actions/publish-wheels/action.yml b/actions/publish-wheels/action.yml index 8dfcde3..4fc98f1 100644 --- a/actions/publish-wheels/action.yml +++ b/actions/publish-wheels/action.yml @@ -57,6 +57,35 @@ inputs: required: false default: '' + gpl-sources-artifact: + description: > + Name of an artifact (produced by an earlier job in the caller workflow) + containing a gpl-sources.tar file with bundled GPL source packages + (e.g. a manylinux toolchain's src.rpm, collected via + collect-gpl-sources). When set, it is downloaded, published + as an asset on the GitHub Release named by gpl-sources-release-tag, + and linked in the docs PR opened by update_doc.py. Leave empty to skip + GPL sources publishing entirely. + required: false + default: '' + + gpl-sources-release-tag: + description: > + Git tag for the GitHub Release that hosts the gpl-sources-artifact + asset, e.g. "numpy-v2.5.1". Required when gpl-sources-artifact is set; + the release is created if it doesn't already exist. + required: false + default: '' + + gpl-sources-description: + description: > + Short parenthetical describing what's bundled in the GPL sources + artifact, e.g. "gcc". Rendered in the docs comment as + "... bundled GPL libraries (gcc)". Only used when gpl-sources-artifact + is set. + required: false + default: '' + runs: using: 'composite' steps: @@ -83,6 +112,40 @@ runs: skip-existing: ${{ inputs.skip-existing }} twine-version: ${{ inputs.twine-version }} + - name: Download GPL sources artifact + if: inputs.gpl-sources-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.gpl-sources-artifact }} + path: gpl-sources + + - name: Publish GPL sources as a GitHub Release asset + if: inputs.gpl-sources-artifact != '' + shell: bash + env: + GH_TOKEN: ${{ inputs.gh-token }} + run: | + set -euo pipefail + + tag="${{ inputs.gpl-sources-release-tag }}" + if [[ -z "$tag" ]]; then + echo "::error::gpl-sources-release-tag must be set when gpl-sources-artifact is used." + exit 1 + fi + + asset="gpl-sources/gpl-sources.tar" + + if gh release view "$tag" --repo "${{ github.repository }}" >/dev/null 2>&1; then + gh release upload "$tag" "$asset" --repo "${{ github.repository }}" --clobber + else + gh release create "$tag" "$asset" \ + --repo "${{ github.repository }}" \ + --title "$tag" \ + --notes "Bundled GPL source packages for $tag." + fi + + echo "GPL_SOURCES_URL=https://github.com/${{ github.repository }}/releases/download/$tag/gpl-sources.tar" >> "$GITHUB_ENV" + - uses: actions/setup-python@v5 with: python-version: '3' @@ -96,4 +159,6 @@ runs: env: GH_TOKEN: ${{ inputs.gh-token }} ARTIFACTS_PATH: ${{ inputs.artifact-path }} + GPL_SOURCES_URL: ${{ env.GPL_SOURCES_URL }} + GPL_SOURCES_DESCRIPTION: ${{ inputs.gpl-sources-description }} run: python3 ci_scripts/update_doc.py diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py index 2bfcc9f..086a2f3 100644 --- a/ci_scripts/update_doc.py +++ b/ci_scripts/update_doc.py @@ -28,6 +28,8 @@ DOCS_DIR = Path("docs/packages") PACKAGES_FILE = Path("ci_scripts/packages.txt") ARTIFACTS_PATH = os.environ.get("ARTIFACTS_PATH", "dist") +GPL_SOURCES_URL = os.environ.get("GPL_SOURCES_URL") +GPL_SOURCES_DESCRIPTION = os.environ.get("GPL_SOURCES_DESCRIPTION", "").strip() def find_wheel_file(path): @@ -102,6 +104,18 @@ def extract_metadata_from_whl(whl_path): } +def render_gpl_sources_comment(): + """ + Render a doc comment linking to a permanently-hosted GPL sources + artifact (e.g. a manylinux toolchain's src.rpm), if the calling workflow + published one for this build. + """ + if not GPL_SOURCES_URL: + return None + suffix = f" ({GPL_SOURCES_DESCRIPTION})" if GPL_SOURCES_DESCRIPTION else "" + return f"`Link <{GPL_SOURCES_URL}>`__ to sources of bundled GPL libraries{suffix}" + + def find_patch_dir(slug, version): """ Look for a `patches//` directory as described in @@ -121,7 +135,7 @@ def yaml_line(key, value): ).rstrip("\n") -def render_new_yaml(slug, source_code, license, version, patch_dir): +def render_new_yaml(slug, source_code, license, version, patch_dir, comment=None): """Render a brand-new docs/packages/.yaml for a package's first version.""" lines = [yaml_line("package-name", slug)] if source_code: @@ -131,10 +145,12 @@ def render_new_yaml(slug, source_code, license, version, patch_dir): lines.append(f" - {yaml_line('version', version)}") if patch_dir is not None: lines.append(" patched:") + if comment: + lines.append(f" {yaml_line('comment', comment)}") return "\n".join(lines) + "\n" -def append_version(content, package_data, version, license, patch_dir): +def append_version(content, package_data, version, license, patch_dir, comment=None): """ Append a new version entry to the end of an existing package YAML file's `versions:` list, preserving the rest of the file byte-for-byte. @@ -153,6 +169,8 @@ def append_version(content, package_data, version, license, patch_dir): lines.append(" patched:") if license and license != top_level_license: lines.append(f" {yaml_line('license', license)}") + if comment: + lines.append(f" {yaml_line('comment', comment)}") return content.rstrip("\n") + "\n" + "\n".join(lines) + "\n" @@ -200,17 +218,20 @@ def main(): slug = normalize_name(display_name) patch_dir = find_patch_dir(slug, version) + comment = render_gpl_sources_comment() yaml_path = DOCS_DIR / f"{slug}.yaml" is_new = not yaml_path.exists() if is_new: yaml_path.write_text( - render_new_yaml(slug, source_code, license, version, patch_dir) + render_new_yaml(slug, source_code, license, version, patch_dir, comment) ) else: content = yaml_path.read_text() package_data = yaml.safe_load(content) or {} - updated = append_version(content, package_data, version, license, patch_dir) + updated = append_version( + content, package_data, version, license, patch_dir, comment + ) if updated is None: print(f"{slug} {version} is already documented; nothing to do") return diff --git a/docs/development.md b/docs/development.md index bfb4b84..5d14ea6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -256,6 +256,57 @@ If either point is not met, we should follow the [Patching a Project](#patching-a-project) process for patching our build, and submit an issue and/or PR upstream to help them comply with license requirements as well. +### Publishing GPL Sources + +Some builds statically or dynamically link against GPL-licensed components +that aren't part of the upstream project itself, but come from our build +environment - most commonly the toolchain baked into the manylinux_riscv64 +container (e.g. `gcc`). As the distributor of the resulting wheel, we need to +make the corresponding source available permanently, not just for as long as +a CI job's artifacts happen to be retained. + +Use the `collect-gpl-sources` action in a job alongside +`build_wheels` to pull the same container image and bundle the source RPM(s) +for one or more installed packages into a `gpl-sources.tar` artifact: + +``` +gpl_sources: + name: Collect GPL sources (gcc) for ${{ inputs.version }} + runs-on: ubuntu-24.04-riscv + steps: + - name: Collect gcc source RPM from manylinux_riscv64 + uses: riseproject-dev/python-wheels/actions/collect-gpl-sources@main + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@v7 + with: + name: -${{ inputs.version }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error +``` + +Pin `MANYLINUX_RISCV64_IMAGE` as a workflow-level `env` and pass it to both +`CIBW_MANYLINUX_RISCV64_IMAGE` in the build job and this job, so the sources +collected actually correspond to the toolchain that produced the wheels (see +`build-numpy.yml` for a complete example). + +Then add `gpl_sources` to the `publish` job's `needs:`, and pass the artifact +through to `publish-wheels`: + +``` +gpl-sources-artifact: -${{ inputs.version }}-gpl-sources +gpl-sources-release-tag: -v${{ inputs.version }} +gpl-sources-description: gcc +``` + +`publish-wheels` publishes the tar as a permanent asset on a GitHub Release +(created if it doesn't already exist) and passes its download URL to +`ci_scripts/update_doc.py`, which renders it as a `comment:` on the new +version entry automatically - no manual doc edit needed. + ### Adding Builds for Rust Packages Modules which are cross-compiled from Rust to Python typically use