diff --git a/.github/actions/detect-affected-packages/action.yml b/.github/actions/detect-affected-packages/action.yml new file mode 100644 index 000000000..3ca5e8399 --- /dev/null +++ b/.github/actions/detect-affected-packages/action.yml @@ -0,0 +1,67 @@ +name: Detect affected packages +description: > + Detects packages affected by a no-bump push to main, i.e. any file changed + under packages// between two commits, excluding packages whose + pyproject.toml version was also bumped in the same range (those release via + release-trigger instead; see docs/versioning.md) and packages removed in the + range (nothing left to build). + + Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` is + reachable. + +inputs: + before: + description: The base commit SHA to compare against (e.g. github.event.before). + required: true + after: + description: The head commit SHA to compare to. Defaults to the checked-out HEAD. + required: false + default: HEAD + +outputs: + count: + description: Number of affected packages. + value: ${{ steps.filter.outputs.count }} + packages: + description: > + JSON array of affected package directory names, e.g. + ["overture-schema-common"]. Suitable as a matrix input. + value: ${{ steps.filter.outputs.packages }} + +runs: + using: composite + steps: + - name: List changed package directories + id: changed-dirs + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 + with: + base_sha: ${{ inputs.before }} + sha: ${{ inputs.after }} + files: packages/** + dir_names: true + dir_names_max_depth: 2 + matrix: true + + # Independent of the step above (neither reads the other's output); runs + # after it only so both feed the final filter step back to back. + - name: Diff package versions + id: diff + uses: ./.github/actions/diff-package-versions + with: + before: ${{ inputs.before }} + after: ${{ inputs.after }} + + - name: Detect affected packages + id: filter + shell: bash + env: + # all_modified_files (not all_changed_files) so deletions count too: a + # file removed from a package without a version bump still changes + # the built wheel. + CHANGED_DIRS: ${{ steps.changed-dirs.outputs.all_modified_files }} + DIFF: ${{ steps.diff.outputs.diff }} + run: | + set -euo pipefail + jq -n --argjson version_diff "$DIFF" --argjson changed_dirs "$CHANGED_DIRS" \ + '{version_diff: $version_diff, changed_dirs: $changed_dirs}' \ + | python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-affected-packages/detect_affected_packages.py b/.github/actions/detect-affected-packages/detect_affected_packages.py new file mode 100644 index 000000000..a4da5c62c --- /dev/null +++ b/.github/actions/detect-affected-packages/detect_affected_packages.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +""" +Detect packages affected by a no-bump push to main. + +Composes `package_versions.py diff`'s version-diff JSON with a JSON array of +changed package directories (tj-actions/changed-files' `dir_names` output) to +find packages whose directory changed without also changing their +pyproject.toml version. Those need an internal `.postN` build (see +docs/versioning.md). Packages with a version bump in the same range are +excluded because they release via `release-trigger` instead, and removed +packages are excluded because there is nothing left to build. + +Reads a single JSON object from stdin: {"version_diff": [...], "changed_dirs": +[...]}. Run from the repository root: + + jq -n \\ + --argjson version_diff "$(python3 package_versions.py diff BEFORE AFTER)" \\ + --argjson changed_dirs '["packages/overture-schema-common"]' \\ + '{version_diff: $version_diff, changed_dirs: $changed_dirs}' \\ + | python3 detect_affected_packages.py + +Prints `$GITHUB_OUTPUT` lines on stdout (progress goes to stderr): + + count= Number of affected packages. + packages= JSON array of affected package directory names, e.g. + ["overture-schema-common"]. Suitable as a matrix input. + +Exit status: + 0 Success (including the no-affected case). + 1 Malformed input: a changed_dirs entry isn't packages/. +""" + +import json +import sys + + +def _package_name(path: str) -> str: + """Extract from a packages/ changed-directory entry.""" + parts = path.split("/", 1) + if len(parts) < 2: + raise ValueError( + f"Expected a 'packages/' changed-directory entry, got {path!r}. " + "Check dir_names_max_depth on the changed-files step." + ) + return parts[1] + + +def main() -> None: + payload = json.load(sys.stdin) + version_diff = payload["version_diff"] + changed_paths = payload["changed_dirs"] + + bumped = {change["package"] for change in version_diff if change["after"] is not None} + removed = {change["package"] for change in version_diff if change["after"] is None} + + changed = {_package_name(path) for path in changed_paths} + affected = sorted(changed - bumped - removed) + + for package in sorted(changed): + if package in removed: + print(f"{package}: removed. Skipping.", file=sys.stderr) + elif package in bumped: + print( + f"{package}: version bumped, handled by release-trigger. " + "Skipping internal build.", + file=sys.stderr, + ) + else: + print(f"{package}: changed, no bump -> internal build.", file=sys.stderr) + + if not affected: + print("No affected packages (no unreleased changes to publish).", file=sys.stderr) + + print(f"count={len(affected)}") + print(f"packages={json.dumps(affected)}") + + +if __name__ == "__main__": + main() diff --git a/.github/actions/detect-version-bumps/action.yml b/.github/actions/detect-version-bumps/action.yml index 22cab9ab0..eed3460cc 100644 --- a/.github/actions/detect-version-bumps/action.yml +++ b/.github/actions/detect-version-bumps/action.yml @@ -31,16 +31,14 @@ runs: steps: - name: Diff package versions id: diff - shell: bash - env: - BEFORE: ${{ inputs.before }} - run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" HEAD \ - > "${RUNNER_TEMP}/package-version-diff.json" + uses: ./.github/actions/diff-package-versions + with: + before: ${{ inputs.before }} - name: Filter to releasable bumps id: filter shell: bash + env: + DIFF: ${{ steps.diff.outputs.diff }} run: | - python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" \ - < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT" + echo "$DIFF" | python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-version-bumps/detect_version_bumps.py b/.github/actions/detect-version-bumps/detect_version_bumps.py index fabdd9aef..957e31c92 100644 --- a/.github/actions/detect-version-bumps/detect_version_bumps.py +++ b/.github/actions/detect-version-bumps/detect_version_bumps.py @@ -17,8 +17,9 @@ - Released versions must be plain `..`; PEP 440 variants like `1.2.3rc4` fail loudly. - A version decrease fails: it must never land on main. - - Added packages (`before` null) and removed packages (`after` null) are - not releases; they are skipped. + - Added packages (`before` null) count as a bump: a package's first + version releases to PyPI like any other. Removed packages (`after` + null) are not releases; they are skipped. The `detect-version-bumps` action composes this with `package_versions.py`, which owns reading versions from git (and enforces the major-bump cascade @@ -64,13 +65,19 @@ def main() -> None: before_raw = change["before"] after_raw = change["after"] - if before_raw is None or after_raw is None: - info(f"{package}: added or removed, not a release. Skipping.") + if after_raw is None: + info(f"{package}: removed, not a release. Skipping.") continue - before = semver(package, before_raw) after = semver(package, after_raw) + if before_raw is None: + info(f"{package}: new package at {after_raw} (bump)") + bumps.append({"package": package, "version": after_raw, "tag": f"{package}-v{after_raw}"}) + continue + + before = semver(package, before_raw) + if after < before: errors.append(f"{package}: {before_raw} -> {after_raw}") continue diff --git a/.github/actions/diff-package-versions/action.yml b/.github/actions/diff-package-versions/action.yml new file mode 100644 index 000000000..8ad20d8f9 --- /dev/null +++ b/.github/actions/diff-package-versions/action.yml @@ -0,0 +1,46 @@ +name: Diff package versions +description: > + Diffs packages/*/pyproject.toml versions between two commits (see + package_versions.py), reading blobs directly from git. Enforces the + major-bump cascade: a package whose direct workspace dependency takes a + major bump must take one itself; the action fails otherwise. + + Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` + is reachable. + +inputs: + before: + description: The base commit SHA to compare against (e.g. github.event.before). + required: true + after: + description: The head commit SHA to compare to. Defaults to the checked-out HEAD. + required: false + default: HEAD + +outputs: + count: + description: Number of packages with a version change (added, removed, or bumped). + value: ${{ steps.diff.outputs.count }} + diff: + description: > + JSON array of {"package", "before", "after"} objects, topologically + sorted (dependencies before dependents). `before`/`after` are null when + the package didn't exist at that commit. + value: ${{ steps.diff.outputs.diff }} + +runs: + using: composite + steps: + - name: Diff package versions + id: diff + shell: bash + env: + BEFORE: ${{ inputs.before }} + AFTER: ${{ inputs.after }} + run: | + python3 "${GITHUB_ACTION_PATH}/package_versions.py" diff "$BEFORE" "$AFTER" \ + > "${RUNNER_TEMP}/package-version-diff.json" + { + echo "count=$(jq 'length' "${RUNNER_TEMP}/package-version-diff.json")" + echo "diff=$(jq -c . "${RUNNER_TEMP}/package-version-diff.json")" + } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/scripts/package_versions.py b/.github/actions/diff-package-versions/package_versions.py similarity index 96% rename from .github/workflows/scripts/package_versions.py rename to .github/actions/diff-package-versions/package_versions.py index c13e7a6f2..2064f00b7 100644 --- a/.github/workflows/scripts/package_versions.py +++ b/.github/actions/diff-package-versions/package_versions.py @@ -3,9 +3,11 @@ """ Diff per-package versions between two git commits. -Run from the repository root: +Run with the repository root as the working directory (git commands need it +as CWD); the script itself can live anywhere, e.g. the `diff-package-versions` +action invokes it via `$GITHUB_ACTION_PATH`: - python3 package_versions.py diff + python3 path/to/package_versions.py diff Reads each `packages/*/pyproject.toml` blob directly from git at both commits (no checkout switching, no environment sync) and prints the packages whose diff --git a/.github/workflows/check-python-code.yaml b/.github/workflows/check-python-code.yaml index b73f78960..c76f252fb 100644 --- a/.github/workflows/check-python-code.yaml +++ b/.github/workflows/check-python-code.yaml @@ -29,13 +29,13 @@ jobs: strategy: fail-fast: false matrix: - # Default resolution exercises the committed lock against every - # supported Python minor version. The lowest-direct cell pins each - # direct dependency to its declared floor (see UV_RESOLUTION below) - # and runs only on the Python floor, since the resolved-low pyspark - # 3.4 wheels exist for 3.10/3.11 only. + # locked exercises the committed lock against every supported Python + # minor version. The lowest-direct cell pins each direct dependency + # to its declared floor (see UV_RESOLUTION below) and runs only on + # the Python floor, since the resolved-low pyspark 3.4 wheels exist + # for 3.10/3.11 only. python: ["3.10", "3.11", "3.12", "3.13", "3.14"] - resolution: [default] + resolution: [locked] include: - python: "3.10" resolution: lowest-direct @@ -80,11 +80,11 @@ jobs: run: echo "UV_RESOLUTION=lowest-direct" >> "$GITHUB_ENV" # Fail fast if uv.lock is stale (e.g. a pyproject.toml version bump - # landed without a matching `uv lock` run). Only for the default + # landed without a matching `uv lock` run). Only for the locked # resolution cells -- lowest-direct intentionally re-resolves away # from the committed lock, so `--locked` would always fail there. - name: Configure lock check - if: matrix.resolution == 'default' + if: matrix.resolution == 'locked' run: echo "UV_LOCKED=1" >> "$GITHUB_ENV" - name: Run make check diff --git a/.github/workflows/compute-versions-dry-run.yaml b/.github/workflows/compute-versions-dry-run.yaml deleted file mode 100644 index a5892f21d..000000000 --- a/.github/workflows/compute-versions-dry-run.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: Compute versions (dry run) - -# Runs on pushes to vnext and main. Computes and logs the version that would -# be published for each affected package — but does not build or publish. -# Also runs (read-only) on PRs that touch the compute-version/code-artifact -# composite actions or this workflow itself, as a smoke test for those. -# Remove this workflow once Phase 3 publish workflows are live (see -# https://github.com/OvertureMaps/schema/issues/509). - -on: - # Real usage: logs the version that would be published for the actual - # branch context. - push: - branches: [main, vnext] - paths: - - '**/pyproject.toml' - # Test usage: smoke-tests the compute-version/code-artifact composite - # actions (and this workflow) against a placeholder context on PRs that - # touch them, so wiring regressions surface before merge. - pull_request: - paths: - - '.github/actions/compute-version/**' - - '.github/actions/code-artifact/**' - - '.github/workflows/compute-versions-dry-run.yaml' - workflow_dispatch: - inputs: - context: - description: "Version context to simulate" - type: choice - options: [vnext, main] - default: vnext - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - discover: - name: Discover context and packages${{ github.event_name == 'pull_request' && ' (test)' || '' }} - if: github.event.repository.full_name == github.repository - runs-on: ubuntu-latest - outputs: - context: ${{ steps.context.outputs.value }} - packages: ${{ steps.packages.outputs.value }} - steps: - - name: Check out code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Determine context - id: context - env: - INPUT_CONTEXT: ${{ inputs.context }} - REF_NAME: ${{ github.ref_name }} - run: | - if [ -n "$INPUT_CONTEXT" ]; then - echo "value=$INPUT_CONTEXT" >> "$GITHUB_OUTPUT" - elif [ "$REF_NAME" = "vnext" ]; then - echo "value=vnext" >> "$GITHUB_OUTPUT" - else - echo "value=main" >> "$GITHUB_OUTPUT" - fi - - - name: Discover packages - id: packages - run: | - packages=$(for pkg_dir in packages/overture-schema*/; do - [ -f "${pkg_dir}/pyproject.toml" ] && basename "$pkg_dir" - done | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "value=${packages}" >> "$GITHUB_OUTPUT" - - compute-versions: - name: Compute version (${{ matrix.package }})${{ github.event_name == 'pull_request' && ' (test)' || '' }} - needs: discover - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # Required for OIDC authentication to AWS - strategy: - fail-fast: false - matrix: - package: ${{ fromJson(needs.discover.outputs.packages) }} - - steps: - - name: Install uv - uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0 - with: - version: latest - - - name: Check out code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_ReadOnly - role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} - - - name: Get CodeArtifact credentials - id: ca - uses: ./.github/actions/code-artifact - - # Delegates to the same composite action Phase 3 publish workflows will - # use, so the version formula only has one implementation to keep correct. - - name: Compute version - id: compute - uses: ./.github/actions/compute-version - with: - package: ${{ matrix.package }} - context: ${{ needs.discover.outputs.context }} - index_url: ${{ steps.ca.outputs.index_url }} - - - name: Report computed version - env: - PACKAGE: ${{ matrix.package }} - CONTEXT: ${{ needs.discover.outputs.context }} - VERSION: ${{ steps.compute.outputs.version }} - run: | - echo "## Computed version: \`${PACKAGE}\`" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Context: \`${CONTEXT}\` " >> "$GITHUB_STEP_SUMMARY" - echo "Version: \`${VERSION}\`" >> "$GITHUB_STEP_SUMMARY" - echo " ${PACKAGE} → ${VERSION}" diff --git a/.github/workflows/main-publish.yaml b/.github/workflows/main-publish.yaml new file mode 100644 index 000000000..6e1981a85 --- /dev/null +++ b/.github/workflows/main-publish.yaml @@ -0,0 +1,174 @@ +name: Main publish + +# Runs on every push to main that touches packages/**. For each package whose +# directory changed WITHOUT a version bump, computes an internal build version +# (main context; see .github/actions/compute-version) and publishes it to +# CodeArtifact. Bumped packages release via release-trigger + release-publish +# instead (see docs/versioning.md) -- publishing a .postN for them here too +# would be redundant. +# +# Also runs (read-only) on PRs that touch the compute-version/code-artifact/ +# detect-affected-packages composite actions or this workflow itself, as a +# smoke test for their wiring: the version is computed but nothing is +# published. + +on: + push: + branches: [main] + paths: + - packages/** + pull_request: + paths: + - '.github/actions/compute-version/**' + - '.github/actions/code-artifact/**' + - '.github/actions/detect-affected-packages/**' + - '.github/workflows/main-publish.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # A real push must never be cancelled by a later one: every no-bump push + # needs its own .postN build recorded in CodeArtifact. Only the PR smoke + # test is safe to supersede. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + detect: + name: Detect affected packages${{ github.event_name == 'pull_request' && ' (test)' || '' }} + if: github.event.repository.full_name == github.repository + runs-on: ubuntu-slim + permissions: + contents: read # Read pyproject.toml/file history to detect affected packages + outputs: + count: ${{ steps.discover.outputs.count || steps.detect.outputs.count }} + packages: ${{ steps.discover.outputs.packages || steps.detect.outputs.packages }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + # A PR touching the composite actions rarely also touches packages/**, + # so the real affected-package diff below would come back empty and + # skip the smoke test entirely. Test every package's wiring instead. + - name: Discover all packages (test) + id: discover + if: github.event_name == 'pull_request' + run: | + packages=$(for pkg_dir in packages/overture-schema*/; do + [ -f "${pkg_dir}/pyproject.toml" ] && basename "$pkg_dir" + done | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "count=$(echo "$packages" | jq 'length')" >> "$GITHUB_OUTPUT" + echo "packages=${packages}" >> "$GITHUB_OUTPUT" + + - name: Detect affected packages + id: detect + if: github.event_name != 'pull_request' + uses: ./.github/actions/detect-affected-packages + with: + before: ${{ github.event.before }} + + publish: + name: Publish ${{ matrix.package }}${{ github.event_name == 'pull_request' && ' (test)' || '' }} + needs: detect + if: needs.detect.outputs.count != '0' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # Required for OIDC authentication to AWS + strategy: + fail-fast: false # One package's failure must not block sibling publishes + matrix: + package: ${{ fromJSON(needs.detect.outputs.packages) }} + steps: + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: latest + + - name: Check out code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Sync code to make packages visible to Python + run: uv sync --locked --all-packages + + # PR smoke test only queries CodeArtifact (via compute-version, below) + # and never publishes -- the publish step is skipped for pull_request. + # Assume the read-only role there so a wiring mistake can't turn the + # smoke test into a real publish. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_${{ github.event_name == 'pull_request' && 'ReadOnly' || 'Publish' }} + role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} + + - name: Get CodeArtifact credentials + id: ca + uses: ./.github/actions/code-artifact + + # Delegates to the same composite action the PR smoke test exercises, so + # the version formula only has one implementation to keep correct. + - name: Compute version + id: compute + uses: ./.github/actions/compute-version + with: + package: ${{ matrix.package }} + context: main + index_url: ${{ steps.ca.outputs.index_url }} + + # overture-schema-pyspark ships generated validation expressions that + # are not committed to git; its packages//scripts/prebuild.sh + # regenerates them before packaging. Every other package has no + # prebuild.sh, so this step is a no-op for them. + - name: Run package's prebuild script, if any + env: + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + run: | + set -euo pipefail + script="packages/${PACKAGE}/scripts/prebuild.sh" + if [ -f "$script" ]; then + echo "Running ${script}" + bash "$script" + fi + + - name: Stamp computed version + # Rewrites pyproject.toml only in this job's temporary checkout, never + # committed: the released .. stays human-owned, + # this just labels the artifact this job builds and publishes. + env: + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + VERSION: ${{ steps.compute.outputs.version }} # zizmor: ignore[template-injection] + run: uv version "${VERSION}" --package "${PACKAGE}" --frozen + + - name: Build ${{ matrix.package }} ${{ steps.compute.outputs.version }} + env: + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + run: uv build --package "${PACKAGE}" + + - name: Publish ${{ matrix.package }} ${{ steps.compute.outputs.version }} to CodeArtifact + if: github.event_name != 'pull_request' + env: + CA_TOKEN: ${{ steps.ca.outputs.token }} + CA_PUBLISH_URL: ${{ steps.ca.outputs.publish_url }} + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + VERSION: ${{ steps.compute.outputs.version }} # zizmor: ignore[template-injection] + run: | + set -euo pipefail + wheel="dist/${PACKAGE//-/_}-${VERSION}-py3-none-any.whl" + if [ ! -f "$wheel" ]; then + echo " Wheel file [$wheel] not found. Aborting!" + exit 1 + fi + tarball="dist/${PACKAGE//-/_}-${VERSION}.tar.gz" + if [ ! -f "$tarball" ]; then + echo " Source tarball file [$tarball] not found. Aborting!" + exit 1 + fi + uv publish "$wheel" "$tarball" \ + -t "${CA_TOKEN}" \ + --publish-url "${CA_PUBLISH_URL}" diff --git a/.github/workflows/publish-python-packages.yaml b/.github/workflows/publish-python-packages.yaml deleted file mode 100644 index 8b2de5ea0..000000000 --- a/.github/workflows/publish-python-packages.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: Publish Python packages to PyPI - -on: - push: - branches: [main] - paths: - - '**/pyproject.toml' - workflow_dispatch: - inputs: - aws_iam_role_name: - description: The name of the IAM role to assume for accessing CodeArtifact - type: string - required: false - default: GithubActions_Schema_CodeArtifact_Publish - domain: - description: The CodeArtifact domain name - type: string - required: false - default: overture-pypi - repository: - description: The CodeArtifact repository name - type: string - required: false - default: overture - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check: - name: Check for changes - if: github.event.repository.full_name == github.repository - uses: ./.github/workflows/reusable-check-python-package-versions.yaml - permissions: - contents: read # Required for checkout in reusable workflow - id-token: write # Required for OIDC in reusable workflow to check AWS CodeArtifact - with: - before_commit: ${{ github.event.before }} - after_commit: ${{ github.event.after }} - - publish: - name: Publish - needs: [check] - if: github.event.repository.full_name == github.repository && needs.check.outputs.num_changed_packages > 0 - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # Required for OIDC authentication to AWS - - strategy: - matrix: - include: ${{ fromJson(needs.check.outputs.changed_packages) }} - steps: - - name: Install uv - uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0 - with: - version: latest - - - name: Check out code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Sync code to make packages visible to Python - run: uv sync --locked --all-packages - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_Publish - role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} - - - name: Get CodeArtifact credentials - id: get-code-artifact-params - uses: ./.github/actions/code-artifact - - # overture-schema-pyspark ships generated validation expressions that are - # not committed to git -- they are regenerated on demand from the Pydantic - # models. `uv build` packages whatever is on disk under the module root, so - # the tree must be generated here before the build, or the published wheel - # ships without its `expressions/generated/` modules and validate_model() - # discovers nothing to run. - - name: Generate PySpark expressions before build - if: matrix.package == 'overture-schema-pyspark' - run: make generate-pyspark - - - name: Publish package ${{ matrix.package }} version ${{ matrix.after }} to PyPI - env: - CA_TOKEN: ${{ steps.get-code-artifact-params.outputs.token }} - CA_PUBLISH_URL: ${{ steps.get-code-artifact-params.outputs.publish_url }} - PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] - BEFORE: ${{ matrix.before }} # zizmor: ignore[template-injection] - AFTER: ${{ matrix.after }} # zizmor: ignore[template-injection] - run: | - printf 'Publishing package %s version %s to PyPI (previous version %s)...\n' "$PACKAGE" "$AFTER" "$BEFORE" - uv build --package "$PACKAGE" - wheel="dist/${PACKAGE//-/_}-${AFTER}-py3-none-any.whl" - if [ ! -f "$wheel" ]; then - echo " Wheel file [$wheel] not found. Aborting!" - exit 1 - fi - # Guard against a silently empty wheel: if the generate step above ever - # regresses, overture-schema-pyspark would publish with no validation - # expressions. Fail loudly instead of shipping a hollow package. - if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ - ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then - echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" - exit 1 - fi - tarball="dist/${PACKAGE//-/_}-${AFTER}.tar.gz" - if [ ! -f "$tarball" ]; then - echo " Source tarball file [$tarball] not found. Aborting!" - exit 1 - fi - uv publish "$wheel" "$tarball" \ - -t "${CA_TOKEN}" \ - --publish-url "${CA_PUBLISH_URL}" diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml new file mode 100644 index 000000000..618632ede --- /dev/null +++ b/.github/workflows/release-publish.yaml @@ -0,0 +1,209 @@ +name: Release publish + +# Triggered when a GitHub Release is published (created by release-trigger.yaml +# using the overture-release-publisher app token -- see #637 -- so the native +# `release: published` event actually fires; GITHUB_TOKEN would not). +# +# Builds the released package at its released version and publishes it to +# public PyPI via Trusted Publishing (OIDC) + attestations. The version-bump +# PR review is the approval gate; nothing further blocks the publish once a +# release exists. +# +# Each package gets its own pypi-release-/test-pypi- +# environment pair, used to scope the Trusted Publisher identity: a PyPI +# Trusted Publisher's identity is (repo, workflow filename, environment), so +# a shared name across packages would let only one package's project name +# ever bind to it (see #653). Neither environment is an approval gate: the +# version-bump PR review gates real releases, and workflow_dispatch already +# requires repo write access to trigger at all, gate enough for a +# disposable .dev0 priming publish. +# +# workflow_dispatch is a manual priming tool, not a release path: it always +# publishes a synthetic .dev0, never the real version, so it +# can safely convert a package's Trusted Publisher from "pending" to "normal" +# (see docs.pypi.org/trusted-publishers/creating-a-project-through-oidc) +# ahead of that package's actual v2.0 release, on either Test PyPI or real +# PyPI. This exists to work around PyPI's per-account rate limit on pending +# publishers (applies to both pypi.org and test.pypi.org independently, ~3 at +# a time per our testing): register a batch, prime them via dispatch to +# convert them, then register the next batch. +# +# TEMPORARY (see #688): the `target: pypi` option below, and everything +# gated on `inputs.target == 'pypi'`, exists only to prime real PyPI's +# Trusted Publishers for the initial v2.0 launch. Tear it out once every +# package has published for real at least once -- `target: test-pypi` (the +# default) is the only permanent path and stays indefinitely for ongoing +# pipeline verification. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + package: + description: Package directory name to prime (e.g. overture-schema-common) + required: true + type: string + target: + description: Where to publish the priming release + required: false + type: choice + default: test-pypi + # TODO(#688): drop the `pypi` option (and this input entirely, once + # test-pypi is the only choice) after all packages are primed. + options: + - test-pypi + - pypi + +permissions: + contents: read + +concurrency: + # One release event per package version; never cancel an in-flight publish. + # workflow_dispatch runs key off the chosen package instead of a tag. + group: ${{ github.workflow }}-${{ github.event.release.tag_name || inputs.package }} + cancel-in-progress: false + +jobs: + parse: + name: Parse package and version + if: github.event_name == 'workflow_dispatch' || github.event.repository.full_name == github.repository + runs-on: ubuntu-slim + outputs: + package: ${{ steps.parse.outputs.package }} + version: ${{ steps.parse.outputs.version }} + steps: + # Checked out unconditionally (both arms) so the package-existence check + # below applies to a bad release tag too, not just a bad workflow_dispatch + # input. Same ref selection the publish job below uses, so both jobs + # build from the same commit for a given event. + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.release.tag_name }} + persist-credentials: false + + - name: Install uv + if: github.event_name == 'workflow_dispatch' + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: false + + - name: Parse package and version + id: parse + env: + EVENT_NAME: ${{ github.event_name }} + TAG: ${{ github.event.release.tag_name }} + INPUT_PACKAGE: ${{ inputs.package }} + run: | + set -euo pipefail + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + package="$INPUT_PACKAGE" + elif [[ "$TAG" =~ ^(.+)-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + package="${BASH_REMATCH[1]}" + version="${BASH_REMATCH[2]}" + else + echo "::error::Release tag '${TAG}' is not -v..; nothing to publish." + exit 1 + fi + + if [[ ! -d "packages/${package}" ]]; then + echo "::error::No such package: packages/${package}" + exit 1 + fi + + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + # .devN is real PEP 440 (unlike the .postN+context.sha local + # versions main-publish.yaml ships, which PyPI/Test PyPI reject + # outright): it never collides with the eventual human-owned + # release version, and default resolvers ignore dev releases, so + # it's safe to actually publish without affecting consumers. Used + # only to prime a package's Trusted Publisher (see #653); never + # the real v2.0 launch artifact. + version="$(cd "packages/${package}" && uv version --short).dev0" + fi + + echo "package=${package}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + + publish: + # TODO(#688): the "PyPI (priming dispatch)" branch below goes away with + # the pypi target. + name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to ${{ github.event_name != 'workflow_dispatch' && 'PyPI' || inputs.target == 'pypi' && 'PyPI (priming dispatch)' || 'Test PyPI' }} + needs: parse + runs-on: ubuntu-latest + # See the header comment for what gates which environment. + # TODO(#688): the `inputs.target == 'pypi' && ...` branch goes away with + # the pypi target -- dispatch will only ever mean Test PyPI at that + # point. + environment: ${{ (github.event_name != 'workflow_dispatch' || inputs.target == 'pypi') && format('pypi-release-{0}', needs.parse.outputs.package) || format('test-pypi-{0}', needs.parse.outputs.package) }} + permissions: + contents: read + id-token: write # Required for PyPI Trusted Publishing (OIDC) + env: + PACKAGE: ${{ needs.parse.outputs.package }} + VERSION: ${{ needs.parse.outputs.version }} + steps: + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: latest + enable-cache: false # Publish job builds and ships immediately; no cache to poison downstream runs + + - name: Check out released commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # target_commitish can be a branch name, which may have moved past + # the release; the tag itself pins the exact released commit. + ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.release.tag_name }} + persist-credentials: false + + - name: Sync code to make packages visible to Python + run: uv sync --locked --all-packages + + # overture-schema-pyspark's scripts/prebuild.sh regenerates its + # not-committed expressions before packaging; other packages have none. + - name: Run package's prebuild script, if any + run: | + set -euo pipefail + script="packages/${PACKAGE}/scripts/prebuild.sh" + if [ -f "$script" ]; then + echo "Running ${script}" + bash "$script" + fi + + # Fails fast if the checked-out commit's version doesn't match the + # release tag. uv has no built-in check for this (astral-sh/uv#9653). + # Only meaningful for a real release: workflow_dispatch intentionally + # publishes a synthetic .devN (see the parse job), not the on-disk + # version, so there's nothing to verify here on that path. + - name: Verify on-disk version matches the release + if: github.event_name != 'workflow_dispatch' + run: | + set -euo pipefail + on_disk="$(cd "packages/${PACKAGE}" && uv version --short)" + if [ "$on_disk" != "$VERSION" ]; then + echo "::error::packages/${PACKAGE} is at version ${on_disk}, but the release tag says ${VERSION}. Aborting!" + exit 1 + fi + + # workflow_dispatch publishes a synthetic .dev0, not the + # on-disk version; stamp it here so the built artifact actually matches + # what gets published. Never committed, this checkout is temporary. + - name: Stamp dispatched version + if: github.event_name == 'workflow_dispatch' + run: uv version "${VERSION}" --package "${PACKAGE}" --frozen + + - name: Build ${{ env.PACKAGE }} ${{ env.VERSION }} + run: uv build --package "${PACKAGE}" + + - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to ${{ github.event_name != 'workflow_dispatch' && 'PyPI' || inputs.target == 'pypi' && 'PyPI (priming dispatch)' || 'Test PyPI' }} + # TODO(#688): once the pypi target is gone, this simplifies back to + # `github.event_name == 'workflow_dispatch'` (dispatch always means + # Test PyPI at that point). + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 + with: + packages-dir: dist/ + attestations: true + repository-url: ${{ github.event_name == 'workflow_dispatch' && inputs.target != 'pypi' && 'https://test.pypi.org/legacy/' || '' }} + skip-existing: ${{ github.event_name == 'workflow_dispatch' }} # Idempotent reruns of a priming dispatch; never true for a real release diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index b8fcd09c6..437fe75df 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -16,10 +16,14 @@ name: Publish GitHub release # is a release. No-bump merges publish internal `.postN` builds instead (see # .github/actions/compute-version). # -# NOTE: releases are created with GITHUB_TOKEN, which by design does NOT trigger -# further workflow runs. The Phase 3 publish workflow (#509) must therefore be -# triggered by an app/PAT token here or via repository_dispatch, a plain -# `on: release: published` listener will not fire for these releases. +# Releases are created with the overture-release-publisher app's installation +# token, not GITHUB_TOKEN: GITHUB_TOKEN-created releases don't fire downstream +# `release: published` events, which release-publish.yaml needs to pick up +# each release individually. See #637 for provisioning that app. +# +# The app's PEM lives in AWS Secrets Manager (omf-github-terraform), fetched +# here via the narrowly-scoped gha-releaser-secrets-reader OIDC role -- same +# pattern as the safe-settings and project-manager app PEMs. on: push: @@ -62,12 +66,37 @@ jobs: if: needs.detect.outputs.count != '0' runs-on: ubuntu-slim permissions: - contents: write # Required to create releases and their tags + contents: read # App token below carries the actual release-creation grant + id-token: write # Required for OIDC authentication to AWS strategy: fail-fast: false # One package's failure must not block sibling releases matrix: include: ${{ fromJSON(needs.detect.outputs.bumps) }} steps: + # Narrow OIDC role (omf-github-terraform's oidc-aws.tf) that can only + # read the releaser app's PEM secret. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::816069134238:role/gha-releaser-secrets-reader + role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} + + # Exports the PEM as env.RELEASE_PUBLISHER_APP_PEM, masked (incl. multi-line). + - name: Fetch releaser PEM from Secrets Manager + uses: aws-actions/aws-secretsmanager-get-secrets@2cb1a461cbd4865ac4299648312e4704c646cd53 # v3.0.1 + with: + secret-ids: | + RELEASE_PUBLISHER_APP_PEM, omf-github-terraform/releaser/pem + + - name: Generate app token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: Iv23lijru2e660v1zJQO # overture-release-publisher app ID, not sensitive + private-key: ${{ env.RELEASE_PUBLISHER_APP_PEM }} # zizmor: ignore[secrets-outside-env] + permission-contents: write + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -83,4 +112,6 @@ jobs: # The umbrella package continues the legacy bare tag series as a # vanity tag (no second release); see docs/versioning.md. vanity-tag: ${{ matrix.package == 'overture-schema' && format('v{0}', matrix.version) || '' }} - token: ${{ github.token }} + # App token so the release fires its own `release: published` event + # for release-publish.yaml (GITHUB_TOKEN would not); see #637. + token: ${{ steps.app-token.outputs.token }} diff --git a/.github/workflows/reusable-check-python-package-versions.yaml b/.github/workflows/reusable-check-python-package-versions.yaml index 6f2d68258..9a1be66aa 100644 --- a/.github/workflows/reusable-check-python-package-versions.yaml +++ b/.github/workflows/reusable-check-python-package-versions.yaml @@ -78,19 +78,21 @@ jobs: persist-credentials: false - name: Diff package versions + id: diff + uses: ./.github/actions/diff-package-versions + with: + before: ${{ inputs.before_commit }} + after: ${{ inputs.after_commit }} + + - name: Save changed packages id: save-changes env: - BEFORE: ${{ inputs.before_commit }} - AFTER: ${{ inputs.after_commit }} + DIFF: ${{ steps.diff.outputs.diff }} + COUNT: ${{ steps.diff.outputs.count }} run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ - > /tmp/package-version-diff.json - cat /tmp/package-version-diff.json { - echo 'changed_packages<> "$GITHUB_OUTPUT" - name: Configure AWS credentials @@ -115,9 +117,10 @@ jobs: if: steps.save-changes.outputs.num_changed_packages > 0 env: INDEX_URL: ${{ steps.get-code-artifact-index-url.outputs.index_url }} + DIFF: ${{ steps.diff.outputs.diff }} # zizmor: ignore[template-injection] run: | - jq -c '.[]' /tmp/package-version-diff.json | while read -r entry; do + jq -c '.[]' <<< "$DIFF" | while read -r entry; do package=$(echo "$entry" | jq -r '.package') after=$(echo "$entry" | jq -r '.after') if [ "$after" = "null" ]; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf31d7a4a..2cd33b1db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,6 @@ Thank you for your interest in contributing. -> **The branching and versioning strategy is rolling out in phases.** See the -> [DevOps tracking issue #490](https://github.com/OvertureMaps/schema/issues/490) -> for current status and what is planned next. - ## Where to send your change This repository uses a two-branch model. Target the branch that matches your @@ -49,8 +45,9 @@ gitGraph main → patch or minor release (version bump) A bug fix or minor feature that bumps the version in the PR and builds the -changelog. On merge, `release-trigger` cuts a published GitHub Release and the -new version lands on PyPI, immediately available to consumers. +changelog. On merge, `release-trigger` cuts a published GitHub Release, which +starts the PyPI publish via Trusted Publishing; the version-bump PR review is +the approval, so nothing further gates it before reaching consumers. ```mermaid gitGraph @@ -71,7 +68,7 @@ gitGraph Breaking changes stack on `vnext` until the milestone is ready. Then `vnext` merges into `main` as a regular merge (not a squash), which cuts a published -GitHub Release and puts the new major on PyPI for consumers. +GitHub Release and starts the same PyPI publish as any other release. ```mermaid gitGraph diff --git a/docs/versioning.md b/docs/versioning.md index 31ae9971c..b50dd3713 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -52,6 +52,17 @@ over the `main` one. `vnext` builds publish only once a separate dev repository exists. Local labels are rejected by public PyPI, which keeps internal builds off the public index by construction. +| Event | Workflow | +|-------|----------| +| Push to `main` | [`main-publish.yaml`](../.github/workflows/main-publish.yaml) detects packages changed without a version bump and publishes their `.postN` build to CodeArtifact. | +| Version bump merged to `main` | [`release-trigger.yaml`](../.github/workflows/release-trigger.yaml) cuts a GitHub Release per bumped package. | +| Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI. | +| Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package and a target (Test PyPI or real PyPI), build a synthetic `.dev0` (never the on-disk release version), and publish it. Used to prime a package's PyPI Trusted Publisher from "pending" to "normal" ahead of its real release (see [#653](https://github.com/OvertureMaps/schema/issues/653)); real-PyPI dispatches require `pypi-dispatch-` approval since they skip PR review entirely. | + +`release-trigger` creates releases with the `overture-release-publisher` app's +installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does +not fire the `release: published` event `release-publish` listens for. + ### Workspace dependency floors Intra-repo dependencies follow uv's @@ -172,13 +183,14 @@ changes that package, whether or not it bumps the version. 2. On merge to `main`, `release-trigger` publishes one GitHub Release per bumped package: tag `-v`, notes from that package's `CHANGELOG.md`. -3. Publishing the release starts the PyPI publish, gated by a maintainer - approval. +3. Publishing the release starts the PyPI publish via Trusted Publishing + (OIDC); no further manual approval gates it. The version-bump PR review + is the approval. ```mermaid flowchart LR A[bump + towncrier build
merged to main] --> B[release-trigger:
GitHub Release per package] - B --> C[PyPI publish
maintainer approval] --> D[public PyPI] + B --> C[PyPI publish
Trusted Publishing] --> D[public PyPI] E[no-bump merge] --> F[.postN internal build
CodeArtifact only] ``` diff --git a/packages/overture-schema-pyspark/changelog.d/638.misc.md b/packages/overture-schema-pyspark/changelog.d/638.misc.md new file mode 100644 index 000000000..1087c5785 --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/638.misc.md @@ -0,0 +1 @@ +Moved expression-generation from the shared prebuild step into the package's own `scripts/prebuild.sh`, run by `release-publish.yaml` before `uv build`. CI-only change: no runtime code, dependencies, or wheel contents affected. diff --git a/packages/overture-schema-pyspark/scripts/prebuild.sh b/packages/overture-schema-pyspark/scripts/prebuild.sh new file mode 100755 index 000000000..24753ad74 --- /dev/null +++ b/packages/overture-schema-pyspark/scripts/prebuild.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Regenerates the PySpark validation expressions from the Pydantic models. +# The expressions/generated/ tree is not committed to git, and `uv build` +# packages whatever is on disk under the module root, so this must run +# before building or packaging this package or the wheel ships without it. +# +# Invoked as a package-owned convention: CI runs this generically (see +# .github/workflows/main-publish.yaml and release-publish.yaml) as +# `packages//scripts/prebuild.sh`, if the file exists, before +# `uv build --package `. Neither workflow needs to know pyspark is +# special; every other package simply has no prebuild.sh. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$repo_root" + +output_dir="packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated" +rm -rf "$output_dir" +uv run overture-codegen generate --format pyspark --output-dir "$output_dir" + +# Guard against a silently empty tree: if codegen ever regresses to produce +# nothing, fail loudly here instead of shipping a hollow package to PyPI. +if ! find "$output_dir" -name '*.py' -print -quit | grep -q .; then + echo "::error::No expressions generated under ${output_dir} -- codegen produced nothing." >&2 + exit 1 +fi diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 7af28fa48..3032dabbb 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -1,7 +1,5 @@ """PySpark validation expressions for Overture Maps data.""" -import importlib.util - from ._pyspark_version import pyspark_version_problem # pyspark is an optional extra (the `spark` extra): a bare install lets this @@ -10,14 +8,16 @@ # import of any submodule -- so a bare install gets an actionable message. A # pyspark that is present but broken still raises its own error, because the # imports below run for real. -if importlib.util.find_spec("pyspark") is None: +try: + import pyspark +except ModuleNotFoundError as exc: + if exc.name != "pyspark": + raise raise ModuleNotFoundError( "overture-schema-pyspark requires PySpark, which isn't installed. " "Install it with `pip install overture-schema-pyspark[spark]`, or run " "in an environment that already provides PySpark (e.g. a Spark cluster)." - ) - -import pyspark + ) from exc # Installing without the extra leaves no resolver to enforce the version floor # declared alongside it, so enforce it here, against the PySpark that actually