[HYPERSHELL-240] feat(ci): ephemeral PR environments for OpenShift e2e

.github/component-paths.json

@@ -94,12 +94,19 @@
       "scripts/install-openshell.sh",
       "deploy/base/**",
       "deploy/kind/**",
+      "deploy/openshift/**",
+      "deploy/e2e/**",
       "scripts/kind/**",
+      "scripts/cluster/**",
+      "scripts/ci/**",
       ".github/component-paths.json",
       ".github/scripts/detect-components.sh",
       ".github/workflows/e2e.yml",
+      ".github/workflows/e2e-openshift-main.yml",
       ".github/workflows/unit-tests.yml",
-      ".github/workflows/tests.yml"
+      ".github/workflows/tests.yml",
+      ".github/workflows/pr-environment.yml",
+      ".github/workflows/pr-environment-release.yml"
     ]
   },
   "pr_test": {

.github/workflows/e2e-openshift-main.yml

@@ -0,0 +1,206 @@
+name: E2E OpenShift (main)
+
+# Bring-up-test-tear-down OpenShift e2e against hypershell-ci-main on every
+# push to main. Lives in its own workflow so Tests / E2E on pull_request
+# does not list a skipped "E2E OpenShift (main)" check. Pull-request
+# OpenShift e2e is Tests / E2E / OpenShift in e2e.yml, against the
+# ephemeral per-PR environment.
+#
+# No hypershell-github-oauth Secret is provisioned here, so this is the
+# traditional admin/admin auth path (github_idp_enabled() in
+# scripts/cluster/drivers/openshift.sh is false), not GitHub brokering.
+# Seeding is deferred until this push's Konflux images are swapped in,
+# mirroring e2e-kind's push path. The environment is always torn down so
+# it does not linger on the shared cluster between runs.
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  checks: read
+
+concurrency:
+  group: e2e-openshift-main
+  cancel-in-progress: true
+
+env:
+  KONFLUX_REGISTRY: quay.io/redhat-user-workloads/hcm-eng-prod-tenant/hypershell-main
+  BASELINE_REGISTRY: quay.io/redhat-services-prod/hcm-eng-prod-tenant/hypershell-main
+  OPENSHIFT_NAMESPACE: hypershell-ci-main
+  KUBECONFIG: ${{ github.workspace }}/.kube/config
+
+jobs:
+  e2e-openshift-main:
+    name: E2E OpenShift (main)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Plan images and required Konflux builds
+        id: plan
+        env:
+          PUSH_BEFORE_SHA: ${{ github.event.before }}
+          PUSH_SHA: ${{ github.sha }}
+        run: |
+          baseline_api="${BASELINE_REGISTRY}/hypershell-api-server-main:latest"
+          baseline_cp="${BASELINE_REGISTRY}/hypershell-control-plane-main:latest"
+          baseline_wc="${BASELINE_REGISTRY}/hypershell-web-console-main:latest"
+
+          if [[ -z "${PUSH_BEFORE_SHA}" || "${PUSH_BEFORE_SHA}" =~ ^0+$ ]]; then
+            changed_files="$(git show --pretty=format: --name-only "${PUSH_SHA}")"
+          else
+            changed_files="$(git diff --name-only "${PUSH_BEFORE_SHA}...${PUSH_SHA}")"
+          fi
+
+          api_server=false
+          control_plane=false
+          web_console=false
+          # These patterns MUST mirror each component's on-push Konflux CEL
+          # trigger in .tekton/hypershell-<component>-main-push.yaml.
+          if grep -qE '^components/api-server/|^\.tekton/hypershell-api-server-main-push\.yaml$' <<<"${changed_files}"; then
+            api_server=true
+          fi
+          if grep -qE '^components/control-plane/|^\.tekton/hypershell-control-plane-main-push\.yaml$|^Dockerfile$' <<<"${changed_files}"; then
+            control_plane=true
+          fi
+          if grep -qE '^components/web-console/|^packages/gateway-management-ui/|^\.tekton/hypershell-web-console-main-push\.yaml$' <<<"${changed_files}"; then
+            web_console=true
+          fi
+
+          tag="${PUSH_SHA}"
+          api_img="${baseline_api}"
+          cp_img="${baseline_cp}"
+          wc_img="${baseline_wc}"
+          echo "wait_api_server=${api_server}" >> "${GITHUB_OUTPUT}"
+          echo "wait_control_plane=${control_plane}" >> "${GITHUB_OUTPUT}"
+          echo "wait_web_console=${web_console}" >> "${GITHUB_OUTPUT}"
+
+          if [[ "${api_server}" == "true" ]]; then
+            api_img="${KONFLUX_REGISTRY}/hypershell-api-server-main:${tag}"
+            echo "  api-server -> ${api_img} (waiting for Konflux build)"
+          fi
+          if [[ "${control_plane}" == "true" ]]; then
+            cp_img="${KONFLUX_REGISTRY}/hypershell-control-plane-main:${tag}"
+            echo "  control-plane -> ${cp_img} (waiting for Konflux build)"
+          fi
+          if [[ "${web_console}" == "true" ]]; then
+            wc_img="${KONFLUX_REGISTRY}/hypershell-web-console-main:${tag}"
+            echo "  web-console -> ${wc_img} (waiting for Konflux build)"
+          fi
+
+          echo "api_server_image=${api_img}" >> "${GITHUB_OUTPUT}"
+          echo "control_plane_image=${cp_img}" >> "${GITHUB_OUTPUT}"
+          echo "web_console_image=${wc_img}" >> "${GITHUB_OUTPUT}"
+
+      - name: Install oc, skopeo, and openshell CLI
+        run: |
+          mkdir -p "$(dirname "${KUBECONFIG}")"
+          curl -LsS https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \
+            | sudo tar -xz -C /usr/local/bin oc kubectl
+          oc version --client
+          sudo apt-get update
+          sudo apt-get install -y skopeo
+          skopeo --version
+          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.110/install.sh | OPENSHELL_VERSION=v0.0.110 sh
+          openshell --version
+
+      - name: Log in to the target cluster
+        env:
+          SERVER_URL: ${{ secrets.OPENSHIFT_PR_ENV_SERVER_URL }}
+          TOKEN: ${{ secrets.OPENSHIFT_PR_ENV_TOKEN }}
+        run: |
+          if [[ -z "${SERVER_URL}" || -z "${TOKEN}" ]]; then
+            echo "::error::OPENSHIFT_PR_ENV_SERVER_URL / OPENSHIFT_PR_ENV_TOKEN are not set"
+            exit 1
+          fi
+          oc login --server="${SERVER_URL}" --token="${TOKEN}" >/dev/null
+          echo "Logged in as $(oc whoami) at $(oc whoami --show-server)"
+          mkdir -p "${HOME}/.docker"
+          oc get secret pull-secret -n openshift-config \
+            -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d > "${HOME}/.docker/config.json"
+
+      - name: Deploy / reconcile environment (make openshift-up)
+        env:
+          SKIP_SEED: "true"
+        run: make openshift-up
+
+      - name: Wait for api-server Konflux build
+        if: steps.plan.outputs.wait_api_server == 'true'
+        uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1
+        with:
+          ref: ${{ github.sha }}
+          check-regexp: 'hypershell-api-server-main-on-push$'
+          repo-token: ${{ github.token }}
+          wait-interval: 30
+          checks-discovery-timeout: 900
+          allowed-conclusions: success
+
+      - name: Wait for control-plane Konflux build
+        if: steps.plan.outputs.wait_control_plane == 'true'
+        uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1
+        with:
+          ref: ${{ github.sha }}
+          check-regexp: 'hypershell-control-plane-main-on-push$'
+          repo-token: ${{ github.token }}
+          wait-interval: 30
+          checks-discovery-timeout: 900
+          allowed-conclusions: success
+
+      - name: Wait for web-console Konflux build
+        if: steps.plan.outputs.wait_web_console == 'true'
+        uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1
+        with:
+          ref: ${{ github.sha }}
+          check-regexp: 'hypershell-web-console-main-on-push$'
+          repo-token: ${{ github.token }}
+          wait-interval: 30
+          checks-discovery-timeout: 900
+          allowed-conclusions: success
+
+      - name: Swap component images
+        env:
+          API_SERVER_IMAGE: ${{ steps.plan.outputs.api_server_image }}
+          CONTROL_PLANE_IMAGE: ${{ steps.plan.outputs.control_plane_image }}
+          WEB_CONSOLE_IMAGE: ${{ steps.plan.outputs.web_console_image }}
+        run: bash scripts/ci/swap-openshift-images-by-digest.sh
+
+      - name: Seed platform resources
+        env:
+          SEED_STRICT: "true"
+        run: make openshift-seed
+
+      - name: Run e2e tests
+        env:
+          E2E_INFRA_DRIVER: openshift
+          TERM: dumb
+          NO_COLOR: "1"
+        run: make e2e
+
+      - name: Collect diagnostics
+        if: failure()
+        run: |
+          mkdir -p e2e-diagnostics
+          oc get pods -n "${OPENSHIFT_NAMESPACE}" -o wide 2>&1 | tee e2e-diagnostics/pods.txt || true
+          oc get events --sort-by=.lastTimestamp -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/events.txt || true
+          oc logs --all-containers --prefix --tail=200 -l app=hypershell-api-server -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/api-server.txt || true
+          oc logs --all-containers --prefix --tail=200 -l app=hypershell-controller -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/controller.txt || true
+
+      - name: Upload diagnostics
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: e2e-openshift-main-diagnostics
+          path: e2e-diagnostics/
+          retention-days: 7
+
+      - name: Tear down environment (make openshift-down)
+        if: always()
+        run: make openshift-down

.github/workflows/e2e.yml

@@ -9,15 +9,22 @@ name: E2E
 # workflow runs fully concurrently with Tests as its own top-level entry in
 # the PR checks list, and GitHub Actions `needs:` cannot gate across
 # independently-triggered workflows without a cross-workflow poller, which
-# this repo deliberately avoids. tests.yml owns the triggers, concurrency
-# group, stage sequencing, and change detection: the per-component flags
-# used by plan-images arrive as inputs instead of being detected here. The
-# stage's rolled-up result, together with unit's, is turned into a single
-# required check by the `Tests CI Gate` job in tests.yml, so this workflow needs
-# no summary/gate job of its own. The `checks: read` permission and
-# the wait-on-check-action steps below remain: they gate on Konflux image
-# builds, which are an external system this workflow cannot order with
-# `needs:`.
+# this repo deliberately avoids for Unit vs Checks. tests.yml owns the
+# triggers, concurrency group, stage sequencing, and change detection: the
+# per-component flags used by plan-images arrive as inputs instead of being
+# detected here. The stage's rolled-up result, together with unit's, is
+# turned into a single required check by the `Tests CI Gate` job in
+# tests.yml, so this workflow needs no summary/gate job of its own. The
+# `checks: read` permission and the wait-on-check-action steps below remain:
+# they gate on Konflux image builds (and, for E2E OpenShift, on the separate
+# PR Environment deploy job), which this workflow cannot order with `needs:`.
+#
+# E2E OpenShift lives here (Tests / E2E / OpenShift), not inside the PR
+# Environment workflow. On origin pull_request it polls the "Deploy PR
+# environment" check, then runs the suite against that namespace. Fork PRs,
+# merge_group, and push skip it (no per-PR environment). Push to main uses
+# the separate e2e-openshift-main.yml workflow (bring-up-test-tear-down)
+# so that job does not appear as a skipped check on pull requests.
 on:
   workflow_call:
     inputs:
@@ -645,3 +652,92 @@ jobs:
           name: e2e-diagnostics
           path: e2e-diagnostics/
           retention-days: 7
+
+  # Runs the OpenShift e2e suite against the environment the "PR Environment"
+  # workflow just deployed (ephemeral-pr-environments.spec.md). Lives here so
+  # the check is Tests / E2E / OpenShift, not nested under PR Environment.
+  # Same plan-images / should_run gate as e2e-kind: docs-only and other
+  # e2e-irrelevant PRs skip the suite (the PR Environment still deploys).
+  # Polls the Deploy PR environment check because GitHub Actions `needs:`
+  # cannot cross independently-triggered workflows.
+  e2e-openshift:
+    name: OpenShift
+    needs: plan-images
+    if: >-
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository &&
+      needs.plan-images.outputs.should_run == 'true'
+    runs-on: ubuntu-24.04
+    # Deploy PR environment is allowed 60 minutes; the suite itself is
+    # budgeted like Kind (~45). 105 covers wait-then-run with headroom.
+    timeout-minutes: 105
+    env:
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+      OPENSHIFT_NAMESPACE: hypershell-ci-pr-${{ github.event.pull_request.number }}
+      KUBECONFIG: ${{ github.workspace }}/.kube/config
+    steps:
+      - name: Wait for Deploy PR environment
+        uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-regexp: 'Deploy PR environment$'
+          repo-token: ${{ github.token }}
+          wait-interval: 30
+          checks-discovery-timeout: 900
+          allowed-conclusions: success
+
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install oc and openshell CLI
+        run: |
+          mkdir -p "$(dirname "${KUBECONFIG}")"
+          curl -LsS https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \
+            | sudo tar -xz -C /usr/local/bin oc kubectl
+          oc version --client
+          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.110/install.sh | OPENSHELL_VERSION=v0.0.110 sh
+          openshell --version
+
+      - name: Log in to the target cluster
+        env:
+          SERVER_URL: ${{ secrets.OPENSHIFT_PR_ENV_SERVER_URL }}
+          TOKEN: ${{ secrets.OPENSHIFT_PR_ENV_TOKEN }}
+        run: |
+          if [[ -z "${SERVER_URL}" || -z "${TOKEN}" ]]; then
+            echo "::error::OPENSHIFT_PR_ENV_SERVER_URL / OPENSHIFT_PR_ENV_TOKEN are not set"
+            exit 1
+          fi
+          oc login --server="${SERVER_URL}" --token="${TOKEN}" >/dev/null
+          echo "Logged in as $(oc whoami) at $(oc whoami --show-server)"
+
+      - name: Run OpenShift e2e suite
+        env:
+          E2E_INFRA_DRIVER: openshift
+          E2E_OIDC_GRANT: client_credentials
+          E2E_OIDC_SA_CLIENT_ID: hypershell-e2e
+          TERM: dumb
+          NO_COLOR: "1"
+        run: |
+          secret="$(bash scripts/ci/read-e2e-client-secret.sh)"
+          echo "::add-mask::${secret}"
+          export E2E_OIDC_SA_CLIENT_SECRET="${secret}"
+          bash tests/e2e/e2e-openshell.sh
+
+      - name: Collect diagnostics
+        if: failure()
+        run: |
+          mkdir -p e2e-diagnostics
+          oc get pods -n "${OPENSHIFT_NAMESPACE}" -o wide 2>&1 | tee e2e-diagnostics/pods.txt || true
+          oc get events --sort-by=.lastTimestamp -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/events.txt || true
+          oc logs --all-containers --prefix --tail=200 -l app=hypershell-api-server -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/api-server.txt || true
+          oc logs --all-containers --prefix --tail=200 -l app=hypershell-controller -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/controller.txt || true
+
+      - name: Upload diagnostics
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: pr-env-diagnostics-${{ github.event.pull_request.number }}
+          path: e2e-diagnostics/
+          retention-days: 7