diff --git a/.github/component-paths.json b/.github/component-paths.json index bdc435be1..c7dcd102c 100644 --- a/.github/component-paths.json +++ b/.github/component-paths.json @@ -94,12 +94,19 @@ "scripts/install-openshell.sh", "deploy/base/**", "deploy/kind/**", + "deploy/openshift/**", + "deploy/e2e/**", "scripts/kind/**", + "scripts/cluster/**", + "scripts/ci/**", ".github/component-paths.json", ".github/scripts/detect-components.sh", ".github/workflows/e2e.yml", + ".github/workflows/e2e-openshift-main.yml", ".github/workflows/unit-tests.yml", - ".github/workflows/tests.yml" + ".github/workflows/tests.yml", + ".github/workflows/pr-environment.yml", + ".github/workflows/pr-environment-release.yml" ] }, "pr_test": { diff --git a/.github/workflows/e2e-openshift-main.yml b/.github/workflows/e2e-openshift-main.yml new file mode 100644 index 000000000..0f061f977 --- /dev/null +++ b/.github/workflows/e2e-openshift-main.yml @@ -0,0 +1,206 @@ +name: E2E OpenShift (main) + +# Bring-up-test-tear-down OpenShift e2e against hypershell-ci-main on every +# push to main. Lives in its own workflow so Tests / E2E on pull_request +# does not list a skipped "E2E OpenShift (main)" check. Pull-request +# OpenShift e2e is Tests / E2E / OpenShift in e2e.yml, against the +# ephemeral per-PR environment. +# +# No hypershell-github-oauth Secret is provisioned here, so this is the +# traditional admin/admin auth path (github_idp_enabled() in +# scripts/cluster/drivers/openshift.sh is false), not GitHub brokering. +# Seeding is deferred until this push's Konflux images are swapped in, +# mirroring e2e-kind's push path. The environment is always torn down so +# it does not linger on the shared cluster between runs. + +on: + push: + branches: + - main + +permissions: + contents: read + checks: read + +concurrency: + group: e2e-openshift-main + cancel-in-progress: true + +env: + KONFLUX_REGISTRY: quay.io/redhat-user-workloads/hcm-eng-prod-tenant/hypershell-main + BASELINE_REGISTRY: quay.io/redhat-services-prod/hcm-eng-prod-tenant/hypershell-main + OPENSHIFT_NAMESPACE: hypershell-ci-main + KUBECONFIG: ${{ github.workspace }}/.kube/config + +jobs: + e2e-openshift-main: + name: E2E OpenShift (main) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Plan images and required Konflux builds + id: plan + env: + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_SHA: ${{ github.sha }} + run: | + baseline_api="${BASELINE_REGISTRY}/hypershell-api-server-main:latest" + baseline_cp="${BASELINE_REGISTRY}/hypershell-control-plane-main:latest" + baseline_wc="${BASELINE_REGISTRY}/hypershell-web-console-main:latest" + + if [[ -z "${PUSH_BEFORE_SHA}" || "${PUSH_BEFORE_SHA}" =~ ^0+$ ]]; then + changed_files="$(git show --pretty=format: --name-only "${PUSH_SHA}")" + else + changed_files="$(git diff --name-only "${PUSH_BEFORE_SHA}...${PUSH_SHA}")" + fi + + api_server=false + control_plane=false + web_console=false + # These patterns MUST mirror each component's on-push Konflux CEL + # trigger in .tekton/hypershell--main-push.yaml. + if grep -qE '^components/api-server/|^\.tekton/hypershell-api-server-main-push\.yaml$' <<<"${changed_files}"; then + api_server=true + fi + if grep -qE '^components/control-plane/|^\.tekton/hypershell-control-plane-main-push\.yaml$|^Dockerfile$' <<<"${changed_files}"; then + control_plane=true + fi + if grep -qE '^components/web-console/|^packages/gateway-management-ui/|^\.tekton/hypershell-web-console-main-push\.yaml$' <<<"${changed_files}"; then + web_console=true + fi + + tag="${PUSH_SHA}" + api_img="${baseline_api}" + cp_img="${baseline_cp}" + wc_img="${baseline_wc}" + echo "wait_api_server=${api_server}" >> "${GITHUB_OUTPUT}" + echo "wait_control_plane=${control_plane}" >> "${GITHUB_OUTPUT}" + echo "wait_web_console=${web_console}" >> "${GITHUB_OUTPUT}" + + if [[ "${api_server}" == "true" ]]; then + api_img="${KONFLUX_REGISTRY}/hypershell-api-server-main:${tag}" + echo " api-server -> ${api_img} (waiting for Konflux build)" + fi + if [[ "${control_plane}" == "true" ]]; then + cp_img="${KONFLUX_REGISTRY}/hypershell-control-plane-main:${tag}" + echo " control-plane -> ${cp_img} (waiting for Konflux build)" + fi + if [[ "${web_console}" == "true" ]]; then + wc_img="${KONFLUX_REGISTRY}/hypershell-web-console-main:${tag}" + echo " web-console -> ${wc_img} (waiting for Konflux build)" + fi + + echo "api_server_image=${api_img}" >> "${GITHUB_OUTPUT}" + echo "control_plane_image=${cp_img}" >> "${GITHUB_OUTPUT}" + echo "web_console_image=${wc_img}" >> "${GITHUB_OUTPUT}" + + - name: Install oc, skopeo, and openshell CLI + run: | + mkdir -p "$(dirname "${KUBECONFIG}")" + curl -LsS https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \ + | sudo tar -xz -C /usr/local/bin oc kubectl + oc version --client + sudo apt-get update + sudo apt-get install -y skopeo + skopeo --version + curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.110/install.sh | OPENSHELL_VERSION=v0.0.110 sh + openshell --version + + - name: Log in to the target cluster + env: + SERVER_URL: ${{ secrets.OPENSHIFT_PR_ENV_SERVER_URL }} + TOKEN: ${{ secrets.OPENSHIFT_PR_ENV_TOKEN }} + run: | + if [[ -z "${SERVER_URL}" || -z "${TOKEN}" ]]; then + echo "::error::OPENSHIFT_PR_ENV_SERVER_URL / OPENSHIFT_PR_ENV_TOKEN are not set" + exit 1 + fi + oc login --server="${SERVER_URL}" --token="${TOKEN}" >/dev/null + echo "Logged in as $(oc whoami) at $(oc whoami --show-server)" + mkdir -p "${HOME}/.docker" + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d > "${HOME}/.docker/config.json" + + - name: Deploy / reconcile environment (make openshift-up) + env: + SKIP_SEED: "true" + run: make openshift-up + + - name: Wait for api-server Konflux build + if: steps.plan.outputs.wait_api_server == 'true' + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.sha }} + check-regexp: 'hypershell-api-server-main-on-push$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Wait for control-plane Konflux build + if: steps.plan.outputs.wait_control_plane == 'true' + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.sha }} + check-regexp: 'hypershell-control-plane-main-on-push$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Wait for web-console Konflux build + if: steps.plan.outputs.wait_web_console == 'true' + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.sha }} + check-regexp: 'hypershell-web-console-main-on-push$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Swap component images + env: + API_SERVER_IMAGE: ${{ steps.plan.outputs.api_server_image }} + CONTROL_PLANE_IMAGE: ${{ steps.plan.outputs.control_plane_image }} + WEB_CONSOLE_IMAGE: ${{ steps.plan.outputs.web_console_image }} + run: bash scripts/ci/swap-openshift-images-by-digest.sh + + - name: Seed platform resources + env: + SEED_STRICT: "true" + run: make openshift-seed + + - name: Run e2e tests + env: + E2E_INFRA_DRIVER: openshift + TERM: dumb + NO_COLOR: "1" + run: make e2e + + - name: Collect diagnostics + if: failure() + run: | + mkdir -p e2e-diagnostics + oc get pods -n "${OPENSHIFT_NAMESPACE}" -o wide 2>&1 | tee e2e-diagnostics/pods.txt || true + oc get events --sort-by=.lastTimestamp -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/events.txt || true + oc logs --all-containers --prefix --tail=200 -l app=hypershell-api-server -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/api-server.txt || true + oc logs --all-containers --prefix --tail=200 -l app=hypershell-controller -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/controller.txt || true + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-openshift-main-diagnostics + path: e2e-diagnostics/ + retention-days: 7 + + - name: Tear down environment (make openshift-down) + if: always() + run: make openshift-down diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 632baddf4..592059369 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,15 +9,22 @@ name: E2E # workflow runs fully concurrently with Tests as its own top-level entry in # the PR checks list, and GitHub Actions `needs:` cannot gate across # independently-triggered workflows without a cross-workflow poller, which -# this repo deliberately avoids. tests.yml owns the triggers, concurrency -# group, stage sequencing, and change detection: the per-component flags -# used by plan-images arrive as inputs instead of being detected here. The -# stage's rolled-up result, together with unit's, is turned into a single -# required check by the `Tests CI Gate` job in tests.yml, so this workflow needs -# no summary/gate job of its own. The `checks: read` permission and -# the wait-on-check-action steps below remain: they gate on Konflux image -# builds, which are an external system this workflow cannot order with -# `needs:`. +# this repo deliberately avoids for Unit vs Checks. tests.yml owns the +# triggers, concurrency group, stage sequencing, and change detection: the +# per-component flags used by plan-images arrive as inputs instead of being +# detected here. The stage's rolled-up result, together with unit's, is +# turned into a single required check by the `Tests CI Gate` job in +# tests.yml, so this workflow needs no summary/gate job of its own. The +# `checks: read` permission and the wait-on-check-action steps below remain: +# they gate on Konflux image builds (and, for E2E OpenShift, on the separate +# PR Environment deploy job), which this workflow cannot order with `needs:`. +# +# E2E OpenShift lives here (Tests / E2E / OpenShift), not inside the PR +# Environment workflow. On origin pull_request it polls the "Deploy PR +# environment" check, then runs the suite against that namespace. Fork PRs, +# merge_group, and push skip it (no per-PR environment). Push to main uses +# the separate e2e-openshift-main.yml workflow (bring-up-test-tear-down) +# so that job does not appear as a skipped check on pull requests. on: workflow_call: inputs: @@ -645,3 +652,92 @@ jobs: name: e2e-diagnostics path: e2e-diagnostics/ retention-days: 7 + + # Runs the OpenShift e2e suite against the environment the "PR Environment" + # workflow just deployed (ephemeral-pr-environments.spec.md). Lives here so + # the check is Tests / E2E / OpenShift, not nested under PR Environment. + # Same plan-images / should_run gate as e2e-kind: docs-only and other + # e2e-irrelevant PRs skip the suite (the PR Environment still deploys). + # Polls the Deploy PR environment check because GitHub Actions `needs:` + # cannot cross independently-triggered workflows. + e2e-openshift: + name: OpenShift + needs: plan-images + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + needs.plan-images.outputs.should_run == 'true' + runs-on: ubuntu-24.04 + # Deploy PR environment is allowed 60 minutes; the suite itself is + # budgeted like Kind (~45). 105 covers wait-then-run with headroom. + timeout-minutes: 105 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + OPENSHIFT_NAMESPACE: hypershell-ci-pr-${{ github.event.pull_request.number }} + KUBECONFIG: ${{ github.workspace }}/.kube/config + steps: + - name: Wait for Deploy PR environment + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-regexp: 'Deploy PR environment$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install oc and openshell CLI + run: | + mkdir -p "$(dirname "${KUBECONFIG}")" + curl -LsS https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \ + | sudo tar -xz -C /usr/local/bin oc kubectl + oc version --client + curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.110/install.sh | OPENSHELL_VERSION=v0.0.110 sh + openshell --version + + - name: Log in to the target cluster + env: + SERVER_URL: ${{ secrets.OPENSHIFT_PR_ENV_SERVER_URL }} + TOKEN: ${{ secrets.OPENSHIFT_PR_ENV_TOKEN }} + run: | + if [[ -z "${SERVER_URL}" || -z "${TOKEN}" ]]; then + echo "::error::OPENSHIFT_PR_ENV_SERVER_URL / OPENSHIFT_PR_ENV_TOKEN are not set" + exit 1 + fi + oc login --server="${SERVER_URL}" --token="${TOKEN}" >/dev/null + echo "Logged in as $(oc whoami) at $(oc whoami --show-server)" + + - name: Run OpenShift e2e suite + env: + E2E_INFRA_DRIVER: openshift + E2E_OIDC_GRANT: client_credentials + E2E_OIDC_SA_CLIENT_ID: hypershell-e2e + TERM: dumb + NO_COLOR: "1" + run: | + secret="$(bash scripts/ci/read-e2e-client-secret.sh)" + echo "::add-mask::${secret}" + export E2E_OIDC_SA_CLIENT_SECRET="${secret}" + bash tests/e2e/e2e-openshell.sh + + - name: Collect diagnostics + if: failure() + run: | + mkdir -p e2e-diagnostics + oc get pods -n "${OPENSHIFT_NAMESPACE}" -o wide 2>&1 | tee e2e-diagnostics/pods.txt || true + oc get events --sort-by=.lastTimestamp -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/events.txt || true + oc logs --all-containers --prefix --tail=200 -l app=hypershell-api-server -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/api-server.txt || true + oc logs --all-containers --prefix --tail=200 -l app=hypershell-controller -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/controller.txt || true + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: pr-env-diagnostics-${{ github.event.pull_request.number }} + path: e2e-diagnostics/ + retention-days: 7 diff --git a/.github/workflows/pr-environment-release.yml b/.github/workflows/pr-environment-release.yml new file mode 100644 index 000000000..bb5d3d2f4 --- /dev/null +++ b/.github/workflows/pr-environment-release.yml @@ -0,0 +1,61 @@ +name: Release PR Environment + +# Tears down the ephemeral OpenShift environment when an origin pull request +# merges or closes (ephemeral-pr-environments.spec.md). Lives in its own +# workflow so open/synchronize runs of pr-environment.yml do not list a +# skipped "Release PR environment" check. GitHub's pull_request `closed` +# event covers both merge and close-without-merge. +# +# Shares the per-PR concurrency group with pr-environment.yml so a close +# cancels an in-flight deploy rather than racing it. The out-of-band reaper +# (deploy/e2e/reaper) is the backstop when this event does not fire. + +on: + pull_request: + types: [closed] + +permissions: + contents: read + +concurrency: + group: pr-env-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + release: + name: Release PR environment + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 20 + env: + OPENSHIFT_NAMESPACE: hypershell-ci-pr-${{ github.event.pull_request.number }} + KUBECONFIG: ${{ github.workspace }}/.kube/config + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install oc + run: | + mkdir -p "$(dirname "${KUBECONFIG}")" + curl -LsS https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \ + | sudo tar -xz -C /usr/local/bin oc kubectl + oc version --client + + - name: Log in to the target cluster + env: + SERVER_URL: ${{ secrets.OPENSHIFT_PR_ENV_SERVER_URL }} + TOKEN: ${{ secrets.OPENSHIFT_PR_ENV_TOKEN }} + run: | + if [[ -z "${SERVER_URL}" || -z "${TOKEN}" ]]; then + echo "::error::OPENSHIFT_PR_ENV_SERVER_URL / OPENSHIFT_PR_ENV_TOKEN are not set" + exit 1 + fi + oc login --server="${SERVER_URL}" --token="${TOKEN}" >/dev/null + + # Primary release path. The out-of-band reaper is the backstop when this + # event does not fire. If teardown cannot confirm the release, fail so an + # operator frees the environment. + - name: Remove environment (make openshift-down) + run: make openshift-down diff --git a/.github/workflows/pr-environment.yml b/.github/workflows/pr-environment.yml new file mode 100644 index 000000000..f279de9f1 --- /dev/null +++ b/.github/workflows/pr-environment.yml @@ -0,0 +1,349 @@ +name: PR Environment + +# Ephemeral OpenShift environment for every origin pull request +# (ephemeral-pr-environments.spec.md, HYPERSHELL-240). +# +# Deploys the full stack into a per-PR namespace group on a shared target +# cluster, keeps it continuously deployed to the PR's head commit, and posts a +# single access comment. The environment lives independently of any CI run +# (out-of-band reaper, deploy/e2e/reaper) so a developer can use it as a live +# debug target. +# +# The OpenShift e2e suite does NOT run here. Tests / E2E / OpenShift (e2e.yml, +# invoked by tests.yml) waits on this workflow's "Deploy PR environment" check +# and then runs the suite against the live namespace. Deploy failure and e2e +# failure therefore surface as distinct checks. +# +# Teardown on merge/close lives in pr-environment-release.yml (pull_request +# closed only) so open/synchronize runs do not list a skipped Release check. +# +# Trust boundary: pull_request only (never pull_request_target). Every job is +# additionally guarded on head.repo == this repo, so a fork PR receives no +# cluster credentials and gets no environment. The GitHub org gate + allowlist +# govern interactive LOGIN to an already-deployed origin environment, not deploy. +# +# Required configuration (generate these; the names are stable): +# secrets.OPENSHIFT_PR_ENV_SERVER_URL target cluster API URL +# secrets.OPENSHIFT_PR_ENV_TOKEN CI service-account token (oc login) +# secrets.PR_ENV_GITHUB_OAUTH_CLIENT_ID GitHub OAuth App client id +# secrets.PR_ENV_GITHUB_OAUTH_CLIENT_SECRET GitHub OAuth App client secret +# secrets.PR_ENV_GITHUB_OAUTH_CALLBACK_URL single stable cluster callback URL +# vars.PR_ENV_GITHUB_ORG org gate (default openshift-online) +# vars.PR_ENV_GITHUB_ALLOWLIST extra usernames (comma-separated) +# vars.PR_ENV_TIMEBOX_DAYS timebox in days (default 3) + +on: + pull_request: + types: [opened, reopened, synchronize] + +permissions: + contents: read + checks: read + pull-requests: write + +# Serialize per pull request so two swaps never leave a mixed digest set; a newer +# run cancels an older in-flight one, keeping the comment SHA honest. +concurrency: + group: pr-env-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + KONFLUX_REGISTRY: quay.io/redhat-user-workloads/hcm-eng-prod-tenant/hypershell-main + BASELINE_REGISTRY: quay.io/redhat-services-prod/hcm-eng-prod-tenant/hypershell-main + +jobs: + # --- Plan which component images this PR's environment should run ---------- + plan-images: + name: Plan component images + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + api_server_image: ${{ steps.plan.outputs.api_server_image }} + control_plane_image: ${{ steps.plan.outputs.control_plane_image }} + web_console_image: ${{ steps.plan.outputs.web_console_image }} + wait_api_server: ${{ steps.plan.outputs.wait_api_server }} + wait_control_plane: ${{ steps.plan.outputs.wait_control_plane }} + wait_web_console: ${{ steps.plan.outputs.wait_web_console }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Plan images and required Konflux builds + id: plan + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + # Deploy is unconditional for a PR environment, but only components this + # PR actually built get their on-pr image (waited for and swapped by + # digest). Patterns MUST mirror each component's pull-request Konflux CEL + # trigger, matching the E2E workflow's plan-images job. + changed_files="$(git diff --name-only "${PR_BASE_SHA}...${PR_HEAD_SHA}")" + api_konflux=false; cp_konflux=false; wc_konflux=false + if grep -qE '^components/api-server/|^\.tekton/hypershell-api-server-main-pull-request\.yaml$' <<<"${changed_files}"; then api_konflux=true; fi + if grep -qE '^components/control-plane/|^\.tekton/hypershell-control-plane-main-pull-request\.yaml$|^Dockerfile$' <<<"${changed_files}"; then cp_konflux=true; fi + if grep -qE '^components/web-console/|^packages/gateway-management-ui/|^\.tekton/hypershell-web-console-main-pull-request\.yaml$' <<<"${changed_files}"; then wc_konflux=true; fi + + tag="on-pr-${PR_HEAD_SHA}" + # Empty image refs leave the component on whatever make openshift-up + # deployed (baseline). Only components this PR actually built are + # swapped, so we do not re-roll unchanged deployments every run. + api_img=""; cp_img=""; wc_img="" + wait_api=false; wait_cp=false; wait_wc=false + if [[ "${api_konflux}" == "true" ]]; then api_img="${KONFLUX_REGISTRY}/hypershell-api-server-main:${tag}"; wait_api=true; fi + if [[ "${cp_konflux}" == "true" ]]; then cp_img="${KONFLUX_REGISTRY}/hypershell-control-plane-main:${tag}"; wait_cp=true; fi + if [[ "${wc_konflux}" == "true" ]]; then wc_img="${KONFLUX_REGISTRY}/hypershell-web-console-main:${tag}"; wait_wc=true; fi + + { + echo "api_server_image=${api_img}" + echo "control_plane_image=${cp_img}" + echo "web_console_image=${wc_img}" + echo "wait_api_server=${wait_api}" + echo "wait_control_plane=${wait_cp}" + echo "wait_web_console=${wait_wc}" + } >> "${GITHUB_OUTPUT}" + + # --- Deploy / reconcile the environment ----------------------------------- + deploy: + name: Deploy PR environment + needs: plan-images + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + OPENSHIFT_NAMESPACE: hypershell-ci-pr-${{ github.event.pull_request.number }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_ENV_TIMEBOX_DAYS: ${{ vars.PR_ENV_TIMEBOX_DAYS || '3' }} + KUBECONFIG: ${{ github.workspace }}/.kube/config + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Posted before cluster login, deploy, or e2e. First deploy is a + # no-facts placeholder so the access comment is near the top of the + # timeline; a later reconcile keeps the existing access-fact table and + # only updates the heading to the new commit. + - name: Post initial "deploying" comment + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_ENV_PHASE: deploying + run: bash scripts/ci/upsert-pr-comment.sh + + - name: Install oc, skopeo, and openshell CLI + run: | + mkdir -p "$(dirname "${KUBECONFIG}")" + curl -LsS https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \ + | sudo tar -xz -C /usr/local/bin oc kubectl + oc version --client + sudo apt-get update + sudo apt-get install -y skopeo + skopeo --version + curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.110/install.sh | OPENSHELL_VERSION=v0.0.110 sh + openshell --version + + - name: Log in to the target cluster + env: + SERVER_URL: ${{ secrets.OPENSHIFT_PR_ENV_SERVER_URL }} + TOKEN: ${{ secrets.OPENSHIFT_PR_ENV_TOKEN }} + run: | + if [[ -z "${SERVER_URL}" || -z "${TOKEN}" ]]; then + echo "::error::OPENSHIFT_PR_ENV_SERVER_URL / OPENSHIFT_PR_ENV_TOKEN are not set" + exit 1 + fi + oc login --server="${SERVER_URL}" --token="${TOKEN}" >/dev/null + echo "Logged in as $(oc whoami) at $(oc whoami --show-server)" + # Cluster global pull secret so skopeo/oc image info can inspect the + # private Konflux registry. CI SA is cluster-admin on the PR cluster. + mkdir -p "${HOME}/.docker" + oc get secret pull-secret -n openshift-config \ + -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d > "${HOME}/.docker/config.json" + echo "Installed cluster pull secret for registry inspect" + + # Provision the GitHub-broker Secret BEFORE Keycloak's first boot, so the + # realm import substitutes it into the GitHub identity provider and the + # hypershell-e2e client secret (declarative config-as-data; no post-boot + # admin-API mutation). Fail closed if the OAuth config is unset, before any + # environment is advertised. The Keycloak namespace is pre-created so the + # Secret exists when make openshift-up brings Keycloak up. + - name: Provision GitHub OAuth secret (fail closed) + env: + GITHUB_OAUTH_CLIENT_ID: ${{ secrets.PR_ENV_GITHUB_OAUTH_CLIENT_ID }} + GITHUB_OAUTH_CLIENT_SECRET: ${{ secrets.PR_ENV_GITHUB_OAUTH_CLIENT_SECRET }} + GITHUB_OAUTH_CALLBACK_URL: ${{ secrets.PR_ENV_GITHUB_OAUTH_CALLBACK_URL }} + PR_ENV_GITHUB_ORG: ${{ vars.PR_ENV_GITHUB_ORG || 'openshift-online' }} + PR_ENV_GITHUB_ALLOWLIST: ${{ vars.PR_ENV_GITHUB_ALLOWLIST }} + run: | + missing=() + [[ -n "${GITHUB_OAUTH_CLIENT_ID}" ]] || missing+=("PR_ENV_GITHUB_OAUTH_CLIENT_ID") + [[ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]] || missing+=("PR_ENV_GITHUB_OAUTH_CLIENT_SECRET") + [[ -n "${GITHUB_OAUTH_CALLBACK_URL}" ]] || missing+=("PR_ENV_GITHUB_OAUTH_CALLBACK_URL") + if (( ${#missing[@]} > 0 )); then + echo "::error::GitHub OAuth configuration is incomplete: ${missing[*]}" + exit 1 + fi + kc_ns="${OPENSHIFT_NAMESPACE}-keycloak" + oc create namespace "${kc_ns}" --dry-run=client -o yaml | oc apply -f - >/dev/null + # Reuse the first-boot e2e client secret. Keycloak --import-realm + # stamps hypershell-e2e only on first boot; rotating the Secret on + # synchronize would desync E2E_OIDC_SA_CLIENT_SECRET from the realm. + existing="$(oc get secret hypershell-github-oauth -n "${kc_ns}" \ + -o jsonpath='{.data.e2e-client-secret}' 2>/dev/null || true)" + if [[ -n "${existing}" ]]; then + e2e_secret="$(printf '%s' "${existing}" | base64 -d)" + else + e2e_secret="$(openssl rand -hex 24)" + fi + echo "::add-mask::${e2e_secret}" + # Enable the GitHub identity provider whenever the OAuth App secrets + # are present (this step already fail-closes if they are missing). + # Kind/local/stage have no hypershell-github-oauth Secret, so the + # init container leaves the IdP disabled there. The web-console BFF + # enforces org membership / allowlist after OIDC callback when + # GITHUB_ORG_GATE is set (openshift-up copies org+allowlist from + # this Secret). Denied users get no HyperShell session. + oc create secret generic hypershell-github-oauth -n "${kc_ns}" \ + --from-literal=idp-enabled=true \ + --from-literal=client-id="${GITHUB_OAUTH_CLIENT_ID}" \ + --from-literal=client-secret="${GITHUB_OAUTH_CLIENT_SECRET}" \ + --from-literal=callback-url="${GITHUB_OAUTH_CALLBACK_URL}" \ + --from-literal=org="${PR_ENV_GITHUB_ORG}" \ + --from-literal=allowlist="${PR_ENV_GITHUB_ALLOWLIST}" \ + --from-literal=e2e-client-enabled=true \ + --from-literal=e2e-client-secret="${e2e_secret}" \ + --dry-run=client -o yaml | oc apply -f - >/dev/null + echo "Provisioned hypershell-github-oauth in ${kc_ns} (GitHub IdP enabled)" + + # Deploy unconditionally (idempotent + reconciling). Defer seeding until the + # PR's images are swapped in so the seed exercises this PR's contract. + - name: Deploy / reconcile environment (make openshift-up) + env: + SKIP_SEED: "true" + run: make openshift-up + + # start-dev --import-realm only loads the rendered JSON into an empty + # data dir. A Keycloak pod that already booted (this PR, or a + # synchronize after the oauth Secret changed) keeps the old realm. Stamp + # the oauth secret hash onto the pod template so a change recycles + # Keycloak. make openshift-up sets HYPERSHELL_CONSOLE_HOST on the + # render-realm-config init container, so a recycle re-imports the console + # redirect URIs instead of the localhost defaults (avoids Invalid + # parameter: redirect_uri). Skip when the annotation already matches so + # we do not wipe a live realm for no reason. + - name: Recycle Keycloak when GitHub OAuth secret changes + run: | + kc_ns="${OPENSHIFT_NAMESPACE}-keycloak" + hash="$(oc get secret hypershell-github-oauth -n "${kc_ns}" \ + -o jsonpath='{.data.idp-enabled}{.data.client-id}{.data.e2e-client-enabled}' \ + | sha256sum | awk '{print $1}')" + current="$(oc get deploy/keycloak -n "${kc_ns}" \ + -o jsonpath='{.spec.template.metadata.annotations.hypershell\.redhat\.io/oauth-secret}' || true)" + if [[ "${current}" == "${hash}" ]]; then + echo "Keycloak oauth-secret annotation already matches; skip recycle" + exit 0 + fi + oc patch deploy/keycloak -n "${kc_ns}" --type=merge -p \ + "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"hypershell.redhat.io/oauth-secret\":\"${hash}\"}}}}}" + oc rollout status deploy/keycloak -n "${kc_ns}" --timeout=600s + + # Overwrite the opaque environment id with pr- and stamp the + # timebox. make openshift-up does neither. Fails the job on any error. + - name: Stamp namespace group (identity + timebox) + id: stamp + run: bash scripts/ci/stamp-pr-env.sh + + - name: Wait for api-server Konflux build + if: needs.plan-images.outputs.wait_api_server == 'true' + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-regexp: 'hypershell-api-server-main-on-pull-request$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Wait for control-plane Konflux build + if: needs.plan-images.outputs.wait_control_plane == 'true' + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-regexp: 'hypershell-control-plane-main-on-pull-request$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Wait for web-console Konflux build + if: needs.plan-images.outputs.wait_web_console == 'true' + uses: lewagon/wait-on-check-action@369769072fe522a3a8a85c03c96af1e5242a1994 # v1.9.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-regexp: 'hypershell-web-console-main-on-pull-request$' + repo-token: ${{ github.token }} + wait-interval: 30 + checks-discovery-timeout: 900 + allowed-conclusions: success + + - name: Swap PR images by digest + env: + API_SERVER_IMAGE: ${{ needs.plan-images.outputs.api_server_image }} + CONTROL_PLANE_IMAGE: ${{ needs.plan-images.outputs.control_plane_image }} + WEB_CONSOLE_IMAGE: ${{ needs.plan-images.outputs.web_console_image }} + run: bash scripts/ci/swap-openshift-images-by-digest.sh + + - name: Seed platform resources + env: + SEED_STRICT: "true" + run: make openshift-seed + + # Edit the same comment posted at job start now that the environment is + # ready, so it survives a failing Tests / E2E / OpenShift run and always + # reflects the deployed head commit. + - name: Discover access URLs + id: urls + run: | + kc_ns="${OPENSHIFT_NAMESPACE}-keycloak" + api_host="$(oc get route hypershell-api -n "${OPENSHIFT_NAMESPACE}" -o jsonpath='{.spec.host}' 2>/dev/null || true)" + web_host="$(oc get route hypershell-web-console -n "${OPENSHIFT_NAMESPACE}" -o jsonpath='{.spec.host}' 2>/dev/null || true)" + { + echo "api_url=https://${api_host}" + echo "web_url=https://${web_host}" + echo "console_url=$(oc whoami --show-console 2>/dev/null || echo '')" + echo "cluster_api_url=$(oc whoami --show-server 2>/dev/null || echo '')" + } >> "${GITHUB_OUTPUT}" + + - name: Post / update access comment + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_ENV_UPDATED: ${{ (github.event.action == 'synchronize') && 'true' || 'false' }} + PLATFORM_NS: ${{ steps.stamp.outputs.platform_namespace }} + KEYCLOAK_NS: ${{ steps.stamp.outputs.keycloak_namespace }} + CONSOLE_URL: ${{ steps.urls.outputs.console_url }} + API_URL: ${{ steps.urls.outputs.api_url }} + WEB_URL: ${{ steps.urls.outputs.web_url }} + CLUSTER_API_URL: ${{ steps.urls.outputs.cluster_api_url }} + run: bash scripts/ci/upsert-pr-comment.sh + + - name: Collect diagnostics + if: failure() + run: | + mkdir -p e2e-diagnostics + oc get pods -n "${OPENSHIFT_NAMESPACE}" -o wide 2>&1 | tee e2e-diagnostics/pods.txt || true + oc get events --sort-by=.lastTimestamp -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/events.txt || true + oc logs --all-containers --prefix --tail=200 -l app=hypershell-api-server -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/api-server.txt || true + oc logs --all-containers --prefix --tail=200 -l app=hypershell-controller -n "${OPENSHIFT_NAMESPACE}" 2>&1 | tee e2e-diagnostics/controller.txt || true + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: pr-env-diagnostics-${{ github.event.pull_request.number }} + path: e2e-diagnostics/ + retention-days: 7 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3fa068574..b691d4d07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,10 +4,13 @@ name: Tests # SDK drift) lives in the separate, independently-triggered # .github/workflows/checks.yml so it shows as its own entry in the PR checks # list and runs fully concurrently with this workflow -- GitHub Actions -# `needs:` only works within a single workflow file, so the two cannot gate -# each other without a cross-workflow poller, and this repo deliberately -# avoids that pattern. Each triggers its own `detect-changes` job rather than -# sharing one, which is the cost of that split. +# `needs:` only works within a single workflow file, so Checks cannot gate +# Tests without a cross-workflow poller, and this repo deliberately avoids +# that pattern for Unit vs Checks. Each triggers its own `detect-changes` +# job rather than sharing one, which is the cost of that split. The one +# exception is Tests / E2E / OpenShift, which polls the independent +# "Deploy PR environment" check (PR Environment workflow) because that +# environment is the suite's target. # # Within this workflow, unit and e2e are reusable workflows (on: # workflow_call) wired with native `needs:` gating: e2e joins on unit @@ -119,6 +122,7 @@ jobs: permissions: contents: read checks: read + secrets: inherit with: api_server: ${{ needs.detect-changes.outputs.api_server }} control_plane: ${{ needs.detect-changes.outputs.control_plane }} diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ddf7f293d..276b0abbe 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -91,7 +91,7 @@ reapplies manifests and waits for readiness. Swapped components are preserved. | `make kind-down` | Remove the `hypershell-system` namespace and its resources. Leaves the Kind cluster running. | | `make kind-teardown` | Destroy the Kind cluster and stop cloud-provider-kind. | | `make kind-status` | Show cluster info, pods, services, and which components are swapped. | -| `make kind-seed` | Re-run ManagedCluster, GatewayRelease, ManagedDatabase, and Gateway seeding. `kind-up` already seeds unless `SKIP_SEED=true`. | +| `make kind-seed` | Re-run ManagedCluster, GatewayRelease, ManagedDatabase, and Gateway seeding. Reuses existing named seed resources (`local-kind`, `dev-release`, `openshell-db`, `dev-gateway`) instead of creating duplicates. `kind-up` already seeds unless `SKIP_SEED=true`. | | `make kind-prereqs` | Build the pinned `cloud-provider-kind` binary into `bin/`. `kind-up` runs this; use it alone when the binary is missing. | | `make kind-env` | Print `export` statements for the current Kind make variables. | | `make kind-fix-ports` | Re-establish host port 443 forwarding to the Gateway's ephemeral port. | @@ -383,10 +383,10 @@ command stops with an error. | Target | Use | |--------|-----| | `make openshift-up` | Deploy the stack into the current oc project (`OPENSHIFT_NAMESPACE` override) and companion `${name}-keycloak`. Does not create an OpenShift cluster. Waits for component rollouts, then seeds unless `SKIP_SEED=true`. | -| `make openshift-down` | Delete the platform and Keycloak projects. If project deletion is forbidden, strip HyperShell resources and leave the projects. | +| `make openshift-down` | Delete the platform and Keycloak projects, then delete gateway and ManagedDatabase namespaces labeled `hypershell.redhat.io/instance=`. If project deletion is forbidden, strip HyperShell resources and leave the projects. | | `make openshift-teardown` | Same as `openshift-down`. There is no OpenShift cluster to destroy. | | `make openshift-status` | Show namespaces, pods, Routes, the shared Gateway, and swap state. | -| `make openshift-seed` | Re-run ManagedCluster, GatewayRelease, ManagedDatabase, and Gateway seeding via API and Keycloak Routes from this machine. `openshift-up` already seeds unless `SKIP_SEED=true`. | +| `make openshift-seed` | Re-run ManagedCluster, GatewayRelease, ManagedDatabase, and Gateway seeding via API and Keycloak Routes from this machine. Reuses existing named seed resources (`local-openshift`, `dev-release`, `openshell-db`, `dev-gateway`) instead of creating duplicates. `openshift-up` already seeds unless `SKIP_SEED=true`. | | `make openshift-api-server-up` | Build, push an immutable image to `SWAP_REGISTRY`, and point the API server Deployment at that ref. Requires `SWAP_REGISTRY`. | | `make openshift-api-server-down` | Revert the API server to the baseline registry image. | | `make openshift-control-plane-up` | Build, push, and swap the control plane. | @@ -516,8 +516,14 @@ port). Keycloak embeds this URL as the `iss` claim in tokens, so the issuer passed to `openshell gateway add` must match exactly. This requires host port 443 to be forwarded -- if it isn't, run `make kind-fix-ports` first. -The e2e test (`components/pr-test/e2e-openshell.sh`) uses the same port-forward -fallback when no passthrough route is available. +The legacy OpenShift e2e script (`components/pr-test/e2e-openshell.sh`) uses the +same port-forward fallback when no passthrough route is available. That script is +**deprecated** (see `specs/platform/ephemeral-pr-environments.spec.md`): the +canonical pull-request OpenShift e2e path is the shared harness +`tests/e2e/e2e-openshell.sh` run with `E2E_INFRA_DRIVER=openshift`, driven +automatically by the ephemeral pull-request environment workflow. Prefer the +shared harness for new work; the IBM ROKS variant (`e2e-openshell-roks.sh`) is +unaffected. ### OpenShift (automatic) diff --git a/components/api-server/cmd/hypershell/main_test.go b/components/api-server/cmd/hypershell/main_test.go index 2fdf5c064..9777f6173 100644 --- a/components/api-server/cmd/hypershell/main_test.go +++ b/components/api-server/cmd/hypershell/main_test.go @@ -5,12 +5,22 @@ import ( "testing" ) +// mustLookupFlag returns the named flag, failing the test immediately if it +// is not registered. Isolating the nil check in its own function (rather than +// inline before later dereferences in the caller) keeps staticcheck's SA5011 +// flow analysis from flagging those dereferences as a possible nil pointer. +func mustLookupFlag(t *testing.T, name string) *flag.Flag { + t.Helper() + f := flag.Lookup(name) + if f == nil { + t.Fatalf("glog flag %q is not registered", name) + } + return f +} + func TestEnforceSafeLogVerbosity(t *testing.T) { - verbosity := flag.Lookup("v") - vmodule := flag.Lookup("vmodule") - if verbosity == nil || vmodule == nil { - t.Fatal("glog verbosity flags are not registered") - } + verbosity := mustLookupFlag(t, "v") + vmodule := mustLookupFlag(t, "vmodule") originalVerbosity := verbosity.Value.String() originalVModule := vmodule.Value.String() t.Cleanup(func() { diff --git a/components/api-server/pkg/rbac/authorization.go b/components/api-server/pkg/rbac/authorization.go index fb6a6a005..23fee054b 100644 --- a/components/api-server/pkg/rbac/authorization.go +++ b/components/api-server/pkg/rbac/authorization.go @@ -293,6 +293,14 @@ func isAuthorized(method string, resource string, resourceID string, gatewayID s } if resource == "gateways" && gatewayID == "" { + // Collection GET is allowed for any authenticated user. The list + // handler filters to accessible IDs and returns 200 with an empty + // items array when there are none. Requiring a RoleBinding here + // 403s developers on OpenShift (RBAC_DEFAULT_ROLES empty) and the + // web console shows "Gateways could not be loaded". + if method == http.MethodGet { + return true + } return hasPlatformAdmin(bindings) || len(bindings) > 0 } diff --git a/components/api-server/pkg/rbac/authorization_test.go b/components/api-server/pkg/rbac/authorization_test.go index bd1af234f..abd15d444 100644 --- a/components/api-server/pkg/rbac/authorization_test.go +++ b/components/api-server/pkg/rbac/authorization_test.go @@ -143,8 +143,24 @@ func TestIsAuthorized_RoleBindingsRequireAnyBinding(t *testing.T) { func TestIsAuthorized_NoBindingsDenied(t *testing.T) { bindings := []BindingSummary{} - if isAuthorized(http.MethodGet, "gateways", "", "", bindings, nil) { - t.Error("empty bindings must be denied") + if isAuthorized(http.MethodGet, "gateways", "gw-1", "gw-1", bindings, nil) { + t.Error("empty bindings must not GET a specific gateway") + } + if isAuthorized(http.MethodPost, "gateways", "", "", bindings, nil) { + t.Error("empty bindings must not POST /gateways") + } +} + +func TestIsAuthorized_NoBindingsCanListGateways(t *testing.T) { + // OpenShift sets RBAC_DEFAULT_ROLES empty, so a developer JWT (only + // hypershell-users) syncs no RoleBindings. Collection GET must still be + // allowed: the list handler returns 200 with an empty items array + // (rbac-enforcement Error Response Opacity). Denying the list is 403 + // and the web console shows "Gateways could not be loaded". + bindings := []BindingSummary{} + + if !isAuthorized(http.MethodGet, "gateways", "", "", bindings, nil) { + t.Error("empty bindings must be allowed to GET /gateways (empty list)") } } @@ -434,6 +450,31 @@ func TestAuthorizeApiAllowsBoundUserFromJWTContext(t *testing.T) { } } +func TestAuthorizeApiAllowsUserWithNoBindingsToListGateways(t *testing.T) { + middleware := NewRBACAuthzMiddleware(authorizationLookup{}, AuthzConfig{EnforceRBAC: true}) + + reached := false + router := mux.NewRouter() + router.Handle("/api/hypershell/v1/gateways", middleware.AuthorizeApi(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached = true + w.WriteHeader(http.StatusOK) + }))).Methods(http.MethodGet) + + request := httptest.NewRequest(http.MethodGet, "/api/hypershell/v1/gateways", nil) + token := &jwt.Token{Claims: jwt.MapClaims{"preferred_username": "developer"}} + ctx := context.WithValue(request.Context(), auth.ContextAuthKey, token) + ctx = context.WithValue(ctx, ContextUserIDKey, "user-id") + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request.WithContext(ctx)) + + if !reached { + t.Fatal("authenticated user with no RoleBindings did not reach GET /gateways") + } + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", recorder.Code) + } +} + func TestAuthorizeApiDeniesGatewayCreatorOnUsersList(t *testing.T) { lookup := authorizationLookup{bindings: []BindingSummary{{RoleName: "gateway:creator", Scope: "global"}}} middleware := NewRBACAuthzMiddleware(lookup, AuthzConfig{EnforceRBAC: true}) diff --git a/components/api-server/plugins/users/dao.go b/components/api-server/plugins/users/dao.go index 90a18b891..cf93483c8 100644 --- a/components/api-server/plugins/users/dao.go +++ b/components/api-server/plugins/users/dao.go @@ -76,24 +76,33 @@ func (d *sqlUserDao) Delete(ctx context.Context, id string) error { return nil } +// Upsert atomically inserts or updates by username via INSERT ... ON +// CONFLICT, avoiding the check-then-act race of a separate SELECT + Create/ +// Save: two concurrent first-time provisioning calls for the same brand-new +// username (e.g. a JIT-provisioned OIDC identity's first API request) would +// otherwise both find no existing row, both attempt Create, and the loser +// would get a raw unique-constraint error instead of the persisted user. +// +// The user's ID is client-generated (BeforeCreate, not DB-autoincrement), so +// GORM does not reliably scan the server's post-conflict row back into it: +// on a losing INSERT it can leave the caller's own generated ID in place +// instead of the row that actually persisted. Re-read by username so every +// caller resolves to the one canonical row regardless of who won the race. func (d *sqlUserDao) Upsert(ctx context.Context, user *User) (*User, error) { g2 := (*d.sessionFactory).New(ctx) - var existing User - result := g2.Where("username = ?", user.Username).Take(&existing) - if result.Error == nil { - existing.Email = user.Email - existing.Name = user.Name - if err := g2.Omit(clause.Associations).Save(&existing).Error; err != nil { - db.MarkForRollback(ctx, err) - return nil, err - } - return &existing, nil - } - if err := g2.Omit(clause.Associations).Create(user).Error; err != nil { + if err := g2.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "username"}}, + DoUpdates: clause.AssignmentColumns([]string{"email", "name"}), + }).Omit(clause.Associations).Create(user).Error; err != nil { db.MarkForRollback(ctx, err) return nil, err } - return user, nil + + var persisted User + if err := g2.Take(&persisted, "username = ?", user.Username).Error; err != nil { + return nil, err + } + return &persisted, nil } func (d *sqlUserDao) FindByIDs(ctx context.Context, ids []string) (UserList, error) { diff --git a/components/api-server/plugins/users/integration_test.go b/components/api-server/plugins/users/integration_test.go index 669010de7..90bc8d268 100644 --- a/components/api-server/plugins/users/integration_test.go +++ b/components/api-server/plugins/users/integration_test.go @@ -137,3 +137,34 @@ func TestUserGet_AllowedForAuthorizedCaller(t *testing.T) { Expect(user.Username).NotTo(BeEmpty()) Expect(user.CreatedAt).NotTo(BeNil()) } + +// TestUserUpsert_ConcurrentFirstTimeProvisioning guards against the +// check-then-act race in Upsert: two requests JIT-provisioning the same +// brand-new OIDC identity at once must not let the losing goroutine see a +// raw unique-constraint error instead of the persisted user. +func TestUserUpsert_ConcurrentFirstTimeProvisioning(t *testing.T) { + test.RegisterIntegration(t) + + userService := users.Service(&environments.Environment().Services) + username := fmt.Sprintf("concurrent-user-%d", time.Now().UnixNano()) + + const concurrency = 8 + ids := make([]string, concurrency) + errs := make([]error, concurrency) + done := make(chan int, concurrency) + for i := range concurrency { + go func(i int) { + ids[i], errs[i] = userService.UpsertByUsername(context.Background(), username, nil, nil) + done <- i + }(i) + } + for range concurrency { + <-done + } + + for i := range concurrency { + Expect(errs[i]).NotTo(HaveOccurred()) + Expect(ids[i]).NotTo(BeEmpty()) + Expect(ids[i]).To(Equal(ids[0])) + } +} diff --git a/components/control-plane/internal/gateway/reconciler.go b/components/control-plane/internal/gateway/reconciler.go index 72574b5d6..0a587680e 100644 --- a/components/control-plane/internal/gateway/reconciler.go +++ b/components/control-plane/internal/gateway/reconciler.go @@ -1360,6 +1360,9 @@ func reconcileKeycloakClient(ctx context.Context, opts ReconcileOpts, nsConfig * if err := kc.EnsureDeviceAuthorizationGrant(ctx, existingUUID); err != nil { return fmt.Errorf("reconcile device authorization grant on keycloak client %s: %w", kcClientID, err) } + if err := kc.EnsureE2ETokenExchange(ctx, existingUUID); err != nil { + return fmt.Errorf("reconcile e2e token-exchange on keycloak client %s: %w", kcClientID, err) + } log.Printf("INFO reconciled keycloak client %s (uuid=%s)", kcClientID, existingUUID) } else { clientUUID, err := kc.ProvisionGatewayClient(ctx, kcClientID) diff --git a/components/control-plane/internal/gateway/reconciler_keycloak_test.go b/components/control-plane/internal/gateway/reconciler_keycloak_test.go index 72174889d..66a7e36b8 100644 --- a/components/control-plane/internal/gateway/reconciler_keycloak_test.go +++ b/components/control-plane/internal/gateway/reconciler_keycloak_test.go @@ -30,12 +30,15 @@ func TestReconcileKeycloakClientUpdatesExistingClient(t *testing.T) { t.Errorf("encode token response: %v", err) } case r.URL.Path == "/admin/realms/hypershell/clients" && r.Method == http.MethodGet: - if got := r.URL.Query().Get("clientId"); got != clientID { - t.Errorf("clientId query = %q, want %q", got, clientID) - } w.Header().Set("Content-Type", "application/json") - if _, err := w.Write([]byte(`[{"id":"client-uuid","clientId":"gateway-id"}]`)); err != nil { - t.Errorf("write client list response: %v", err) + if r.URL.Query().Get("clientId") == clientID { + if _, err := w.Write([]byte(`[{"id":"client-uuid","clientId":"gateway-id"}]`)); err != nil { + t.Errorf("write client list response: %v", err) + } + return + } + if _, err := w.Write([]byte(`[]`)); err != nil { + t.Errorf("write empty client list response: %v", err) } case r.URL.Path == "/admin/realms/hypershell/clients/"+clientUUID && r.Method == http.MethodGet: w.Header().Set("Content-Type", "application/json") diff --git a/components/control-plane/internal/keycloak/client.go b/components/control-plane/internal/keycloak/client.go index 6b44301d4..49b846c8a 100644 --- a/components/control-plane/internal/keycloak/client.go +++ b/components/control-plane/internal/keycloak/client.go @@ -15,7 +15,16 @@ import ( "time" ) -const deviceAuthorizationGrantAttribute = "oauth2.device.authorization.grant.enabled" +const ( + deviceAuthorizationGrantAttribute = "oauth2.device.authorization.grant.enabled" + + e2eClientID = "hypershell-e2e" + e2eTokenExchangePolicyName = "hypershell-e2e-token-exchange" + realmManagementClientID = "realm-management" + frontendClientID = "hypershell-frontend" + tokenExchangeScope = "token-exchange" + tokenExchangeDecisionStrategy = "UNANIMOUS" +) // Client wraps the Keycloak Admin REST API for gateway OIDC provisioning. // Configuration and httpClient do not change after construction. The HTTP @@ -57,6 +66,7 @@ func (c *Client) Realm() string { type keycloakClient struct { ID string `json:"id,omitempty"` ClientID string `json:"clientId"` + Enabled bool `json:"enabled"` Attributes map[string]string `json:"attributes,omitempty"` } @@ -111,6 +121,15 @@ func (c *Client) ProvisionGatewayClient(ctx context.Context, gatewayName string) } log.Printf("INFO keycloak: created protocol mappers on client %s", gatewayName) + log.Printf("INFO keycloak: granting %s token-exchange on client %s", e2eClientID, gatewayName) + if err := c.EnsureE2ETokenExchange(ctx, clientUUID); err != nil { + log.Printf("WARN keycloak: token-exchange grant failed for %s, rolling back client: %v", gatewayName, err) + if rollbackErr := c.deleteClientByUUID(ctx, clientUUID); rollbackErr != nil { + log.Printf("WARN keycloak: failed to rollback client %s after token-exchange grant failure: %v", gatewayName, rollbackErr) + } + return "", fmt.Errorf("grant e2e token-exchange: %w", err) + } + return clientUUID, nil } @@ -396,6 +415,261 @@ func (c *Client) EnsureDeviceAuthorizationGrant(ctx context.Context, clientUUID return nil } +type managementPermissions struct { + Enabled bool `json:"enabled"` + Resource string `json:"resource,omitempty"` + ScopePermissions map[string]string `json:"scopePermissions,omitempty"` +} + +type authzPolicy struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +// EnsureE2ETokenExchange grants the hypershell-e2e client FGAP v1 +// token-exchange onto targetClientUUID and, when present, hypershell-frontend. +// Area 4 exchanges onto the per-gateway client; area 9 exchanges onto the +// frontend API audience. Skip unless that client exists and is enabled: a +// present-but-disabled representation (Kind, local OpenShift, hub) must not +// enable admin-fine-grained-authz or attach token-exchange policies on the +// gateway reconcile path. +func (c *Client) EnsureE2ETokenExchange(ctx context.Context, targetClientUUID string) error { + if targetClientUUID == "" { + return fmt.Errorf("target client UUID is required for token-exchange") + } + + e2e, err := c.getClient(ctx, e2eClientID) + if err != nil { + return fmt.Errorf("look up %s client: %w", e2eClientID, err) + } + if e2e == nil { + log.Printf("INFO keycloak: %s client not present; skipping token-exchange grants", e2eClientID) + return nil + } + if !e2e.Enabled { + log.Printf("INFO keycloak: %s client is present but disabled; skipping token-exchange grants", e2eClientID) + return nil + } + e2eUUID := e2e.ID + if e2eUUID == "" { + return fmt.Errorf("keycloak client %s is enabled but has no id", e2eClientID) + } + + rmUUID, err := c.getClientUUID(ctx, realmManagementClientID) + if err != nil { + return fmt.Errorf("look up %s client: %w", realmManagementClientID, err) + } + if rmUUID == "" { + return fmt.Errorf("keycloak client %s not found", realmManagementClientID) + } + + // Enabling FGAP on the target client lazily initializes realm-management's + // authorization resource server. Without it, the policy search/create + // endpoints 404 on a fresh realm, so this must precede the policy work. + targetPermID, err := c.enableTokenExchangePermissions(ctx, targetClientUUID) + if err != nil { + return err + } + + policyID, err := c.ensureE2EClientPolicy(ctx, rmUUID, e2eUUID) + if err != nil { + return err + } + + if err := c.attachPolicyToPermission(ctx, rmUUID, targetPermID, policyID); err != nil { + return fmt.Errorf("grant token-exchange on client %s: %w", targetClientUUID, err) + } + + frontendUUID, err := c.getClientUUID(ctx, frontendClientID) + if err != nil { + return fmt.Errorf("look up %s client: %w", frontendClientID, err) + } + if frontendUUID == "" || frontendUUID == targetClientUUID { + return nil + } + frontendPermID, err := c.enableTokenExchangePermissions(ctx, frontendUUID) + if err != nil { + return err + } + if err := c.attachPolicyToPermission(ctx, rmUUID, frontendPermID, policyID); err != nil { + return fmt.Errorf("grant token-exchange on %s: %w", frontendClientID, err) + } + return nil +} + +func (c *Client) ensureE2EClientPolicy(ctx context.Context, realmMgmtUUID, e2eClientUUID string) (string, error) { + if id, err := c.findE2EClientPolicy(ctx, realmMgmtUUID); err != nil { + return "", err + } else if id != "" { + return id, nil + } + + payload, err := json.Marshal(map[string]interface{}{ + "name": e2eTokenExchangePolicyName, + "logic": "POSITIVE", + "decisionStrategy": tokenExchangeDecisionStrategy, + "clients": []string{e2eClientUUID}, + }) + if err != nil { + return "", fmt.Errorf("marshal %s policy: %w", e2eTokenExchangePolicyName, err) + } + + path := fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/policy/client", c.realm, realmMgmtUUID) + resp, err := c.doRequestRaw(ctx, http.MethodPost, path, payload) + if err != nil { + return "", fmt.Errorf("create %s policy: %w", e2eTokenExchangePolicyName, err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode == http.StatusConflict { + id, err := c.findE2EClientPolicy(ctx, realmMgmtUUID) + if err != nil { + return "", err + } + if id == "" { + return "", fmt.Errorf("create %s policy returned 409 but the policy was not found", e2eTokenExchangePolicyName) + } + return id, nil + } + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("create %s policy returned %d: %s", e2eTokenExchangePolicyName, resp.StatusCode, string(body)) + } + + var created authzPolicy + if err := json.Unmarshal(body, &created); err != nil { + return "", fmt.Errorf("parse %s policy: %w", e2eTokenExchangePolicyName, err) + } + if created.ID == "" { + return "", fmt.Errorf("create %s policy returned no id", e2eTokenExchangePolicyName) + } + return created.ID, nil +} + +func (c *Client) findE2EClientPolicy(ctx context.Context, realmMgmtUUID string) (string, error) { + path := fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/policy/search?name=%s", + c.realm, realmMgmtUUID, url.QueryEscape(e2eTokenExchangePolicyName)) + resp, err := c.doRequestRaw(ctx, http.MethodGet, path, nil) + if err != nil { + return "", fmt.Errorf("search %s policy: %w", e2eTokenExchangePolicyName, err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + + // Keycloak's policy search-by-name returns 404 (not an empty 200/204) when no + // policy of that name exists yet, so treat it as "not found" and let the caller + // create the policy rather than failing the reconcile. + if resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound || len(bytes.TrimSpace(body)) == 0 { + return "", nil + } + if resp.StatusCode >= 400 { + return "", fmt.Errorf("search %s policy returned %d: %s", e2eTokenExchangePolicyName, resp.StatusCode, string(body)) + } + + var found authzPolicy + if err := json.Unmarshal(body, &found); err != nil { + return "", fmt.Errorf("parse %s policy search: %w", e2eTokenExchangePolicyName, err) + } + return found.ID, nil +} + +func (c *Client) attachPolicyToPermission(ctx context.Context, realmMgmtUUID, permID, policyID string) error { + associatedPath := fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/policy/%s/associatedPolicies", + c.realm, realmMgmtUUID, permID) + associatedBody, err := c.doRequest(ctx, http.MethodGet, associatedPath, nil) + if err != nil { + return fmt.Errorf("list token-exchange associated policies: %w", err) + } + var associated []authzPolicy + if len(bytes.TrimSpace(associatedBody)) > 0 { + if err := json.Unmarshal(associatedBody, &associated); err != nil { + return fmt.Errorf("parse token-exchange associated policies: %w", err) + } + } + for _, policy := range associated { + if policy.ID == policyID { + return nil + } + } + + permPath := fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/permission/scope/%s", + c.realm, realmMgmtUUID, permID) + permBody, err := c.doRequest(ctx, http.MethodGet, permPath, nil) + if err != nil { + return fmt.Errorf("get token-exchange permission: %w", err) + } + + var representation map[string]json.RawMessage + if err := json.Unmarshal(permBody, &representation); err != nil { + return fmt.Errorf("parse token-exchange permission: %w", err) + } + + policyIDs := make([]string, 0, len(associated)+1) + for _, policy := range associated { + if policy.ID != "" { + policyIDs = append(policyIDs, policy.ID) + } + } + policyIDs = append(policyIDs, policyID) + rawPolicies, err := json.Marshal(policyIDs) + if err != nil { + return fmt.Errorf("marshal token-exchange policies: %w", err) + } + representation["policies"] = rawPolicies + + if rawStrategy, ok := representation["decisionStrategy"]; !ok || + len(rawStrategy) == 0 || + bytes.Equal(bytes.TrimSpace(rawStrategy), []byte("null")) || + bytes.Equal(bytes.TrimSpace(rawStrategy), []byte(`""`)) { + representation["decisionStrategy"] = json.RawMessage(`"` + tokenExchangeDecisionStrategy + `"`) + } + + body, err := json.Marshal(representation) + if err != nil { + return fmt.Errorf("marshal token-exchange permission: %w", err) + } + if _, err := c.doRequest(ctx, http.MethodPut, permPath, body); err != nil { + return fmt.Errorf("attach token-exchange policy: %w", err) + } + return nil +} + +func (c *Client) enableTokenExchangePermissions(ctx context.Context, targetClientUUID string) (string, error) { + path := fmt.Sprintf("/admin/realms/%s/clients/%s/management/permissions", c.realm, targetClientUUID) + payload, err := json.Marshal(managementPermissions{Enabled: true}) + if err != nil { + return "", fmt.Errorf("marshal management permissions: %w", err) + } + + body, err := c.doRequest(ctx, http.MethodPut, path, payload) + if err != nil { + return "", fmt.Errorf("enable management permissions on client %s: %w", targetClientUUID, err) + } + + var perms managementPermissions + if len(bytes.TrimSpace(body)) > 0 { + if err := json.Unmarshal(body, &perms); err != nil { + return "", fmt.Errorf("parse management permissions for client %s: %w", targetClientUUID, err) + } + } + if permID := perms.ScopePermissions[tokenExchangeScope]; permID != "" { + return permID, nil + } + + getBody, err := c.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return "", fmt.Errorf("get management permissions for client %s: %w", targetClientUUID, err) + } + if err := json.Unmarshal(getBody, &perms); err != nil { + return "", fmt.Errorf("parse management permissions for client %s: %w", targetClientUUID, err) + } + permID := perms.ScopePermissions[tokenExchangeScope] + if permID == "" { + return "", fmt.Errorf("client %s has no token-exchange permission; enable admin-fine-grained-authz:v1", targetClientUUID) + } + return permID, nil +} + func (c *Client) createClientRoles(ctx context.Context, clientUUID string) error { for _, roleName := range []string{"openshell-admin", "openshell-user"} { role := keycloakRole{Name: roleName} @@ -443,23 +717,34 @@ func (c *Client) createProtocolMappers(ctx context.Context, clientUUID, gatewayN return nil } -func (c *Client) getClientUUID(ctx context.Context, clientID string) (string, error) { +func (c *Client) getClient(ctx context.Context, clientID string) (*keycloakClient, error) { path := fmt.Sprintf("/admin/realms/%s/clients?clientId=%s", c.realm, url.QueryEscape(clientID)) respBody, err := c.doRequest(ctx, http.MethodGet, path, nil) if err != nil { - return "", err + return nil, err } var clients []keycloakClient if err := json.Unmarshal(respBody, &clients); err != nil { - return "", fmt.Errorf("parse client list: %w", err) + return nil, fmt.Errorf("parse client list: %w", err) } - for _, kc := range clients { - if kc.ClientID == clientID { - return kc.ID, nil + for i := range clients { + if clients[i].ClientID == clientID { + return &clients[i], nil } } - return "", nil + return nil, nil +} + +func (c *Client) getClientUUID(ctx context.Context, clientID string) (string, error) { + kc, err := c.getClient(ctx, clientID) + if err != nil { + return "", err + } + if kc == nil { + return "", nil + } + return kc.ID, nil } func (c *Client) listClientRoles(ctx context.Context, clientUUID string) ([]keycloakRole, error) { diff --git a/components/control-plane/internal/keycloak/client_test.go b/components/control-plane/internal/keycloak/client_test.go index b6189068c..d4ba91ac1 100644 --- a/components/control-plane/internal/keycloak/client_test.go +++ b/components/control-plane/internal/keycloak/client_test.go @@ -599,3 +599,278 @@ func TestEnsureDeviceAuthorizationGrantSkipsEnabledClient(t *testing.T) { default: } } + +const ( + teTestE2EUUID = "e2e-client-uuid" + teTestRealmMgmtUUID = "realm-management-uuid" + teTestFrontendUUID = "frontend-client-uuid" + teTestTargetUUID = "target-client-uuid" + teTestPolicyUUID = "e2e-policy-uuid" + teTestTargetPerm = "target-te-perm-uuid" + teTestFrontendPerm = "frontend-te-perm-uuid" +) + +type tokenExchangeFake struct { + mu sync.Mutex + + e2ePresent bool + e2eEnabled bool + policyExists bool + alreadyGranted map[string]bool + createdPolicy bool + enabledClients []string + attachedPerms []string + lookedUp []string +} + +func newTokenExchangeServer(t *testing.T, fake *tokenExchangeFake) *httptest.Server { + t.Helper() + clientsPath := fmt.Sprintf("/admin/realms/%s/clients", testRealm) + policySearchPath := fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/policy/search", testRealm, teTestRealmMgmtUUID) + policyCreatePath := fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/policy/client", testRealm, teTestRealmMgmtUUID) + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == deviceTestTokenPath: + writeTokenResponse(t, w, t.Name()) + case r.URL.Path == clientsPath && r.Method == http.MethodGet: + clientID := r.URL.Query().Get("clientId") + fake.mu.Lock() + fake.lookedUp = append(fake.lookedUp, clientID) + e2ePresent := fake.e2ePresent + e2eEnabled := fake.e2eEnabled + fake.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + switch clientID { + case e2eClientID: + if !e2ePresent { + _ = json.NewEncoder(w).Encode([]keycloakClient{}) + return + } + _ = json.NewEncoder(w).Encode([]keycloakClient{{ + ID: teTestE2EUUID, + ClientID: e2eClientID, + Enabled: e2eEnabled, + }}) + case realmManagementClientID: + _ = json.NewEncoder(w).Encode([]keycloakClient{{ID: teTestRealmMgmtUUID, ClientID: realmManagementClientID}}) + case frontendClientID: + _ = json.NewEncoder(w).Encode([]keycloakClient{{ID: teTestFrontendUUID, ClientID: frontendClientID}}) + default: + _ = json.NewEncoder(w).Encode([]keycloakClient{}) + } + case strings.HasSuffix(r.URL.Path, "/management/permissions") && r.Method == http.MethodPut: + targetUUID := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, clientsPath+"/"), "/management/permissions") + permID := teTestTargetPerm + if targetUUID == teTestFrontendUUID { + permID = teTestFrontendPerm + } + fake.mu.Lock() + fake.enabledClients = append(fake.enabledClients, targetUUID) + fake.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(managementPermissions{ + Enabled: true, + Resource: "client-resource-" + targetUUID, + ScopePermissions: map[string]string{tokenExchangeScope: permID}, + }) + case r.URL.Path == policySearchPath && r.Method == http.MethodGet: + fake.mu.Lock() + exists := fake.policyExists + fake.mu.Unlock() + if !exists { + // Real Keycloak returns 404 (not 204) from policy search-by-name + // when no policy of that name exists yet. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "HTTP 404 Not Found"}) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(authzPolicy{ID: teTestPolicyUUID, Name: e2eTokenExchangePolicyName}) + case r.URL.Path == policyCreatePath && r.Method == http.MethodPost: + var payload struct { + Name string `json:"name"` + Clients []string `json:"clients"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("decode policy create: %v", err) + http.Error(w, "invalid payload", http.StatusBadRequest) + return + } + if payload.Name != e2eTokenExchangePolicyName { + t.Errorf("policy name = %q, want %q", payload.Name, e2eTokenExchangePolicyName) + } + if len(payload.Clients) != 1 || payload.Clients[0] != teTestE2EUUID { + t.Errorf("policy clients = %v, want [%s]", payload.Clients, teTestE2EUUID) + } + fake.mu.Lock() + fake.createdPolicy = true + fake.policyExists = true + fake.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(authzPolicy{ID: teTestPolicyUUID, Name: e2eTokenExchangePolicyName}) + case strings.HasSuffix(r.URL.Path, "/associatedPolicies") && r.Method == http.MethodGet: + permID := strings.TrimSuffix(r.URL.Path[strings.LastIndex(r.URL.Path[:len(r.URL.Path)-len("/associatedPolicies")], "/")+1:], "/associatedPolicies") + w.Header().Set("Content-Type", "application/json") + fake.mu.Lock() + granted := fake.alreadyGranted[permID] + fake.mu.Unlock() + if granted { + _ = json.NewEncoder(w).Encode([]authzPolicy{{ID: teTestPolicyUUID, Name: e2eTokenExchangePolicyName}}) + return + } + _ = json.NewEncoder(w).Encode([]authzPolicy{}) + case strings.Contains(r.URL.Path, "/permission/scope/") && r.Method == http.MethodGet: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": strings.TrimPrefix(r.URL.Path, fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/permission/scope/", testRealm, teTestRealmMgmtUUID)), + "name": "token-exchange.permission.client", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + }) + case strings.Contains(r.URL.Path, "/permission/scope/") && r.Method == http.MethodPut: + var representation struct { + Policies []string `json:"policies"` + } + if err := json.NewDecoder(r.Body).Decode(&representation); err != nil { + t.Errorf("decode permission update: %v", err) + http.Error(w, "invalid payload", http.StatusBadRequest) + return + } + if len(representation.Policies) == 0 { + t.Error("permission update omitted policies") + } + found := false + for _, id := range representation.Policies { + if id == teTestPolicyUUID { + found = true + break + } + } + if !found { + t.Errorf("permission policies = %v, want to include %s", representation.Policies, teTestPolicyUUID) + } + permID := strings.TrimPrefix(r.URL.Path, fmt.Sprintf("/admin/realms/%s/clients/%s/authz/resource-server/permission/scope/", testRealm, teTestRealmMgmtUUID)) + fake.mu.Lock() + fake.attachedPerms = append(fake.attachedPerms, permID) + fake.mu.Unlock() + w.WriteHeader(http.StatusCreated) + default: + http.NotFound(w, r) + } + })) +} + +func TestEnsureE2ETokenExchangeGrantsGatewayAndFrontend(t *testing.T) { + t.Parallel() + + fake := &tokenExchangeFake{e2ePresent: true, e2eEnabled: true} + server := newTokenExchangeServer(t, fake) + defer server.Close() + + client := NewClient(server.URL, testRealm, testAdminClientID, t.Name()) + if err := client.EnsureE2ETokenExchange(t.Context(), teTestTargetUUID); err != nil { + t.Fatalf("EnsureE2ETokenExchange() error = %v", err) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if !fake.createdPolicy { + t.Error("did not create the hypershell-e2e token-exchange client policy") + } + if got, want := fake.enabledClients, []string{teTestTargetUUID, teTestFrontendUUID}; !equalStrings(got, want) { + t.Errorf("enabled management permissions on %v, want %v", got, want) + } + if got, want := fake.attachedPerms, []string{teTestTargetPerm, teTestFrontendPerm}; !equalStrings(got, want) { + t.Errorf("attached token-exchange policy on %v, want %v", got, want) + } +} + +func TestEnsureE2ETokenExchangeSkipsMissingE2EClient(t *testing.T) { + t.Parallel() + + fake := &tokenExchangeFake{} + server := newTokenExchangeServer(t, fake) + defer server.Close() + + client := NewClient(server.URL, testRealm, testAdminClientID, t.Name()) + if err := client.EnsureE2ETokenExchange(t.Context(), teTestTargetUUID); err != nil { + t.Fatalf("EnsureE2ETokenExchange() error = %v", err) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.createdPolicy || len(fake.enabledClients) > 0 || len(fake.attachedPerms) > 0 { + t.Errorf("granted token-exchange without hypershell-e2e (enabled=%v attached=%v createdPolicy=%v)", + fake.enabledClients, fake.attachedPerms, fake.createdPolicy) + } +} + +func TestEnsureE2ETokenExchangeSkipsDisabledE2EClient(t *testing.T) { + t.Parallel() + + fake := &tokenExchangeFake{e2ePresent: true, e2eEnabled: false} + server := newTokenExchangeServer(t, fake) + defer server.Close() + + client := NewClient(server.URL, testRealm, testAdminClientID, t.Name()) + if err := client.EnsureE2ETokenExchange(t.Context(), teTestTargetUUID); err != nil { + t.Fatalf("EnsureE2ETokenExchange() error = %v", err) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.createdPolicy || len(fake.enabledClients) > 0 || len(fake.attachedPerms) > 0 { + t.Errorf("granted token-exchange for a disabled hypershell-e2e client (enabled=%v attached=%v createdPolicy=%v)", + fake.enabledClients, fake.attachedPerms, fake.createdPolicy) + } +} + +func TestEnsureE2ETokenExchangeSkipsAlreadyGrantedPermission(t *testing.T) { + t.Parallel() + + fake := &tokenExchangeFake{ + e2ePresent: true, + e2eEnabled: true, + policyExists: true, + alreadyGranted: map[string]bool{ + teTestTargetPerm: true, + teTestFrontendPerm: true, + }, + } + server := newTokenExchangeServer(t, fake) + defer server.Close() + + client := NewClient(server.URL, testRealm, testAdminClientID, t.Name()) + if err := client.EnsureE2ETokenExchange(t.Context(), teTestTargetUUID); err != nil { + t.Fatalf("EnsureE2ETokenExchange() error = %v", err) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if fake.createdPolicy { + t.Error("recreated an existing token-exchange client policy") + } + if len(fake.attachedPerms) != 0 { + t.Errorf("re-attached already granted permissions %v", fake.attachedPerms) + } + if got, want := fake.enabledClients, []string{teTestTargetUUID, teTestFrontendUUID}; !equalStrings(got, want) { + t.Errorf("enabled management permissions on %v, want %v", got, want) + } +} + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range want { + if got[i] != want[i] { + return false + } + } + return true +} diff --git a/components/control-plane/internal/reconciler/reconciler.go b/components/control-plane/internal/reconciler/reconciler.go index 9a7c59c14..256b16fbb 100644 --- a/components/control-plane/internal/reconciler/reconciler.go +++ b/components/control-plane/internal/reconciler/reconciler.go @@ -1955,6 +1955,9 @@ func (r *GatewayReconciler) reconcileExistingGatewayKeycloakClient(ctx context.C if err := r.keycloakClient.EnsureDeviceAuthorizationGrant(ctx, clientUUID); err != nil { return fmt.Errorf("reconcile device authorization grant on Keycloak client %q: %w", clientID, err) } + if err := r.keycloakClient.EnsureE2ETokenExchange(ctx, clientUUID); err != nil { + return fmt.Errorf("reconcile e2e token-exchange on Keycloak client %q: %w", clientID, err) + } log.Printf("INFO reconciled Keycloak client %q (uuid=%q)", clientID, clientUUID) return nil } diff --git a/components/control-plane/internal/reconciler/reconciler_test.go b/components/control-plane/internal/reconciler/reconciler_test.go index 4240ec642..8f97edf18 100644 --- a/components/control-plane/internal/reconciler/reconciler_test.go +++ b/components/control-plane/internal/reconciler/reconciler_test.go @@ -765,8 +765,8 @@ func TestWatchGateways_KeycloakRetryPreservesGatedPayload(t *testing.T) { lookupCalls := mockKC.uuidLookupCalls putCalls := mockKC.putCalled mockKC.mu.Unlock() - if lookupCalls != 2 || putCalls != 1 { - t.Fatalf("Keycloak calls: lookups=%d puts=%d, want one failed lookup followed by one successful lookup and PUT", lookupCalls, putCalls) + if lookupCalls != 3 || putCalls != 1 { + t.Fatalf("Keycloak calls: lookups=%d puts=%d, want one failed lookup, one successful gateway lookup, one e2e presence probe, and one PUT", lookupCalls, putCalls) } if updates := gatewayServer.snapshot(); len(updates) != 0 { t.Fatalf("Keycloak-only retry performed provisioning gRPC updates: %v", updates) diff --git a/components/control-plane/internal/serviceaccountprovisioner/transport_test.go b/components/control-plane/internal/serviceaccountprovisioner/transport_test.go index 977512493..285aa69f4 100644 --- a/components/control-plane/internal/serviceaccountprovisioner/transport_test.go +++ b/components/control-plane/internal/serviceaccountprovisioner/transport_test.go @@ -43,7 +43,12 @@ func TestListenAndServeServesPlaintextProvisionerCalls(t *testing.T) { callCtx, callCancel := context.WithTimeout(ctx, 5*time.Second) defer callCancel() - response, err := client.Provision(callCtx, &pb.ProvisionRequest{Spec: &pb.ServiceAccountSpec{GatewayId: "gateway-id"}}) + // WaitForReady retries past the connection racing ListenAndServe's own + // goroutine binding the listener above: without it, gRPC's default dial + // fails fast on the first "connection refused" instead of retrying up to + // callCtx's deadline, making this test flaky under CI scheduling delay. + response, err := client.Provision(callCtx, &pb.ProvisionRequest{Spec: &pb.ServiceAccountSpec{GatewayId: "gateway-id"}}, + grpc.WaitForReady(true)) if err != nil { t.Fatalf("Provision() over plaintext gRPC error = %v", err) } diff --git a/components/pr-test/e2e-openshell.sh b/components/pr-test/e2e-openshell.sh index 8d5d108b0..bdb27e02d 100755 --- a/components/pr-test/e2e-openshell.sh +++ b/components/pr-test/e2e-openshell.sh @@ -1,6 +1,16 @@ #!/usr/bin/env bash # e2e-openshell.sh - end-to-end test of the OpenShell gateway provisioned by HyperShell. # +# DEPRECATED (ephemeral-pr-environments.spec.md, HYPERSHELL-240): +# This hardcoded-OpenShift pull-request e2e script is superseded by the shared, +# infra-agnostic harness at tests/e2e/e2e-openshell.sh run with +# E2E_INFRA_DRIVER=openshift, which the ephemeral pull-request environment +# workflow drives automatically for every origin pull request. It is kept only +# because some team members still run it directly; do NOT add new test areas +# here (new coverage lands in tests/e2e/). Removal is deferred until that manual +# usage migrates. NOTE: this deprecation does NOT apply to the sibling ROKS +# script (e2e-openshell-roks.sh), which targets IBM ROKS and is out of scope. +# # Proves the full path: HyperShell API → control plane → gateway provisioning # → openshell CLI → sandbox pod creation + interaction. # diff --git a/components/web-console/Dockerfile b/components/web-console/Dockerfile index c9f52945b..642259699 100644 --- a/components/web-console/Dockerfile +++ b/components/web-console/Dockerfile @@ -1,12 +1,14 @@ # syntax=docker/dockerfile:1 # HI pins are the per-arch manifests from hi/nodejs:24.18.1-builder and hi/nodejs:24.18.1. -# TARGETARCH selects both stages so native addons and the node binary match the cluster. +# BUILDARCH selects a native Node toolchain so vite/esbuild are not qemu-emulated. +# TARGETARCH selects production native addons (sodium-native) and the runtime. # Bump amd64 and arm64 together when changing tags. +ARG BUILDARCH ARG TARGETARCH -FROM registry.access.redhat.com/hi/nodejs:24.18.1-builder@sha256:7d3b87dbb6bbf8fc85c865f8a00c5355961dae0e0e4ead15d530af3c8d6e3ebe AS build-amd64 -FROM registry.access.redhat.com/hi/nodejs:24.18.1-builder@sha256:5034714f7a4bdd7423231f880b0863b8e69c5aa9b8f5ab69ed1cd1e76760f0e0 AS build-arm64 -FROM build-${TARGETARCH} AS build +FROM --platform=linux/amd64 registry.access.redhat.com/hi/nodejs:24.18.1-builder@sha256:7d3b87dbb6bbf8fc85c865f8a00c5355961dae0e0e4ead15d530af3c8d6e3ebe AS build-amd64 +FROM --platform=linux/arm64 registry.access.redhat.com/hi/nodejs:24.18.1-builder@sha256:5034714f7a4bdd7423231f880b0863b8e69c5aa9b8f5ab69ed1cd1e76760f0e0 AS build-arm64 +FROM --platform=linux/${BUILDARCH} build-${BUILDARCH} AS build USER root @@ -33,13 +35,33 @@ COPY --chown=${CONTAINER_DEFAULT_USER} packages/operational-dashboard-ui package COPY --chown=${CONTAINER_DEFAULT_USER} components/web-console components/web-console COPY --chown=${CONTAINER_DEFAULT_USER} images/brand images/brand -RUN pnpm run build:web \ +RUN pnpm run build:web + +FROM --platform=linux/${TARGETARCH} build-${TARGETARCH} AS bundle + +USER root + +COPY scripts/bootstrap_pnpm.sh . +RUN ./bootstrap_pnpm.sh + +USER ${CONTAINER_DEFAULT_USER} + +WORKDIR /app + +COPY --from=build --chown=${CONTAINER_DEFAULT_USER} /app/package.json /app/package.json +COPY --from=build --chown=${CONTAINER_DEFAULT_USER} /app/pnpm-lock.yaml /app/pnpm-lock.yaml +COPY --from=build --chown=${CONTAINER_DEFAULT_USER} /app/pnpm-workspace.yaml /app/pnpm-workspace.yaml +COPY --from=build --chown=${CONTAINER_DEFAULT_USER} /app/components /app/components +COPY --from=build --chown=${CONTAINER_DEFAULT_USER} /app/packages /app/packages +COPY --from=build --chown=${CONTAINER_DEFAULT_USER} /app/images /app/images + +RUN pnpm install --frozen-lockfile --prod --filter @openshift-online/hypershell-web-console-bff \ && pnpm --filter @openshift-online/hypershell-web-console-bff deploy --prod /tmp/web-console \ && cp -R components/web-console/build/client /tmp/web-console/public -FROM registry.access.redhat.com/hi/nodejs:24.18.1@sha256:6725f3730de000b6255b97d31e2ef53916f35397b1fb779bdb8d4a3a1c84ad50 AS runtime-amd64 -FROM registry.access.redhat.com/hi/nodejs:24.18.1@sha256:8b3a067f10336acb6798f4d8251882fed107c90116b69a469e02472ecc1c7648 AS runtime-arm64 -FROM runtime-${TARGETARCH} +FROM --platform=linux/amd64 registry.access.redhat.com/hi/nodejs:24.18.1@sha256:6725f3730de000b6255b97d31e2ef53916f35397b1fb779bdb8d4a3a1c84ad50 AS runtime-amd64 +FROM --platform=linux/arm64 registry.access.redhat.com/hi/nodejs:24.18.1@sha256:8b3a067f10336acb6798f4d8251882fed107c90116b69a469e02472ecc1c7648 AS runtime-arm64 +FROM --platform=linux/${TARGETARCH} runtime-${TARGETARCH} LABEL org.opencontainers.image.title="HyperShell web console" \ org.opencontainers.image.description="HyperShell web console BFF and static application" \ @@ -53,7 +75,7 @@ ENV HOST=0.0.0.0 \ STATIC_ROOT=/app/public WORKDIR /app -COPY --from=build /tmp/web-console/ ./ +COPY --from=bundle /tmp/web-console/ ./ EXPOSE 8080 diff --git a/components/web-console/bff/src/app.ts b/components/web-console/bff/src/app.ts index 916686766..afca783a1 100644 --- a/components/web-console/bff/src/app.ts +++ b/components/web-console/bff/src/app.ts @@ -12,7 +12,12 @@ import Fastify, { LogController, } from "fastify"; -import { clearSession, persistTokenSet, registerAuth } from "./auth.js"; +import { + AUTH_DENIED_PAGE_HTML, + clearSession, + persistTokenSet, + registerAuth, +} from "./auth.js"; import { hasDashboardAdminRole } from "./roles.js"; import { browserRuntimeConfig, @@ -154,10 +159,19 @@ function injectRuntimeConfig( } function inlineScriptHashes(document: string): string[] { - const hashes = new Set(); - const scriptPattern = /]*\bsrc=)[^>]*>([\s\S]*?)<\/script>/giu; + return cspHashes( + document, + /]*\bsrc=)[^>]*>([\s\S]*?)<\/script>/giu, + ); +} - for (const match of document.matchAll(scriptPattern)) { +function inlineStyleHashes(document: string): string[] { + return cspHashes(document, /]*>([\s\S]*?)<\/style>/giu); +} + +function cspHashes(document: string, pattern: RegExp): string[] { + const hashes = new Set(); + for (const match of document.matchAll(pattern)) { const body = match[1]; if (body) { hashes.add( @@ -165,7 +179,6 @@ function inlineScriptHashes(document: string): string[] { ); } } - return [...hashes]; } @@ -195,6 +208,7 @@ export async function buildApp( browserRuntimeConfig(config), ); const scriptHashes = inlineScriptHashes(indexDocument); + const styleHashes = inlineStyleHashes(AUTH_DENIED_PAGE_HTML); const app = Fastify({ bodyLimit: 1_048_576, @@ -239,7 +253,7 @@ export async function buildApp( imgSrc: ["'self'", "data:"], objectSrc: ["'none'"], scriptSrc: ["'self'", ...scriptHashes], - styleSrc: ["'self'"], + styleSrc: ["'self'", ...styleHashes], styleSrcAttr: ["'none'"], // The BFF serves plain HTTP behind the deployment TLS terminator. // HTTPS already blocks mixed active content, while this directive diff --git a/components/web-console/bff/src/auth.ts b/components/web-console/bff/src/auth.ts index 29db69933..74223a0dd 100644 --- a/components/web-console/bff/src/auth.ts +++ b/components/web-console/bff/src/auth.ts @@ -3,6 +3,7 @@ import type { FastifyInstance } from "fastify"; import * as oidc from "openid-client"; import type { ServerConfig } from "./config.js"; +import { evaluateGithubOrgGate } from "./github-org-gate.js"; import { createRefresher, sanitizeReturnTo, @@ -132,12 +133,106 @@ export function clearSession(request: { request.tokenSession.delete(); } +/** + * Standalone HTML for `/auth/denied`. Helmet's global CSP is `style-src + * 'self'`, so the inline stylesheet is admitted only via a sha256 hash of + * this document's ` + + +
+

HyperShell

+

Access denied

+

This HyperShell environment is limited to members of the configured GitHub organization and allowlisted usernames.

+

Sign out and try a different GitHub account or contact the maintainers of this project to be added to the allowlist.

+
+ + +`; + /** * Registers OIDC-based authentication on the Fastify instance. * * This sets up encrypted cookie sessions via @fastify/secure-session, performs - * OpenID Connect discovery against the configured issuer, and mounts the four - * auth endpoints (/auth/login, /auth/callback, /auth/logout, /auth/session). + * OpenID Connect discovery against the configured issuer, and mounts the + * auth endpoints (/auth/login, /auth/callback, /auth/denied, /auth/logout, + * /auth/session). * * Call this function only when OIDC configuration is present. It must be called * before route registration so that the session decorator is available to all @@ -234,6 +329,14 @@ export async function registerAuth( reply.redirect(authUrl.toString()); }); + app.get("/auth/denied", async (_request, reply) => { + reply + .code(403) + .header("Cache-Control", "no-store") + .type("text/html; charset=utf-8"); + return AUTH_DENIED_PAGE_HTML; + }); + app.get("/auth/callback", async (request, reply) => { const storedState = request.session.get("state"); const storedNonce = request.session.get("nonce"); @@ -263,17 +366,51 @@ export async function registerAuth( ); const claims = tokens.claims(); + const tokenSet = toTokenSet(tokens); + + // Pull-request environments set GITHUB_ORG_GATE so interactive GitHub + // logins are limited to org members and allowlisted usernames. Sessions + // with no GitHub broker identity (password users) are admitted there. + const username = + typeof claims?.preferred_username === "string" + ? claims.preferred_username + : undefined; + if (config.githubOrgGate && config.oidcIssuer) { + const allowed = await evaluateGithubOrgGate({ + accessToken: tokenSet.accessToken, + allowlistRaw: config.githubUsernameAllowlist, + githubApiOrigin: config.githubApiOrigin ?? "https://api.github.com", + oidcIssuer: config.oidcIssuer, + onLookupError: (error) => { + request.log.warn( + { err: error, preferredUsername: username }, + "GitHub org gate lookup failed", + ); + }, + orgGate: config.githubOrgGate, + username, + }); + if (!allowed) { + request.log.info( + { preferredUsername: username }, + "GitHub org gate denied login", + ); + clearSession(request); + reply.redirect("/auth/denied"); + return; + } + } // Replace login session data with auth session data. Rotate both cookies // so no pre-login value survives (session fixation defense). request.session.regenerate(); request.tokenSession.regenerate(); - persistTokenSet(request, toTokenSet(tokens)); + persistTokenSet(request, tokenSet); if (claims) { request.session.set("sub", claims.sub); - if (typeof claims.preferred_username === "string") { - request.session.set("preferredUsername", claims.preferred_username); + if (username !== undefined) { + request.session.set("preferredUsername", username); } if (typeof claims.email === "string") { request.session.set("email", claims.email); diff --git a/components/web-console/bff/src/config.ts b/components/web-console/bff/src/config.ts index 84e141f73..0b53d150e 100644 --- a/components/web-console/bff/src/config.ts +++ b/components/web-console/bff/src/config.ts @@ -52,6 +52,15 @@ const configSchema = z.object({ .min(1) .default("hypershell-web-console-bff"), OTEL_TRACES_SAMPLE_RATIO: z.coerce.number().min(0).max(1).default(1), + GITHUB_API_ORIGIN: httpOrigin.default("https://api.github.com"), + GITHUB_ORG_GATE: z + .string() + .trim() + .optional() + .transform((value) => + value === undefined || value === "" ? undefined : value, + ), + GITHUB_USERNAME_ALLOWLIST: z.string().optional(), OIDC_CLIENT_ID: z.string().trim().min(1).optional(), OIDC_ISSUER: httpUrl.optional(), OIDC_POST_LOGOUT_REDIRECT_URI: httpUrl.optional(), @@ -102,6 +111,9 @@ export interface TracingConfig { export interface ServerConfig { apiOrigin: string; apiTimeoutMs: number; + githubApiOrigin?: string; + githubOrgGate?: string; + githubUsernameAllowlist?: string; host: string; logLevel: z.infer["LOG_LEVEL"]; nodeEnv: z.infer["NODE_ENV"]; @@ -194,6 +206,9 @@ export function loadConfig( return { apiOrigin: result.data.HYPERSHELL_API_ORIGIN, apiTimeoutMs: result.data.HYPERSHELL_API_TIMEOUT_MS, + githubApiOrigin: result.data.GITHUB_API_ORIGIN, + githubOrgGate: result.data.GITHUB_ORG_GATE, + githubUsernameAllowlist: result.data.GITHUB_USERNAME_ALLOWLIST, host: result.data.HOST, logLevel: result.data.LOG_LEVEL, nodeEnv: result.data.NODE_ENV, diff --git a/components/web-console/bff/src/github-org-gate.ts b/components/web-console/bff/src/github-org-gate.ts new file mode 100644 index 000000000..3f2c670db --- /dev/null +++ b/components/web-console/bff/src/github-org-gate.ts @@ -0,0 +1,366 @@ +const githubRequestTimeoutMs = 8_000; +const githubOrgPageLimit = 5; + +export interface GithubOrgGateInput { + accessToken: string; + allowlistRaw: string | undefined; + fetchImpl?: typeof fetch; + githubApiOrigin: string; + oidcIssuer: string; + onLookupError?: (error: unknown) => void; + orgGate: string; + username: string | undefined; +} + +/** Splits a comma-separated GitHub username allowlist into lowercase names. */ +export function parseUsernameAllowlist(raw: string | undefined): string[] { + if (raw === undefined) { + return []; + } + return raw + .split(",") + .map((entry) => entry.trim().toLowerCase()) + .filter((entry) => entry.length > 0); +} + +/** + * Reports whether a GitHub identity may use a pull-request environment. + * + * Allowlist entries are additive: a listed username is admitted even with no + * org membership. Everyone else must belong to `orgGate`. + */ +export function githubIdentityAllowed(input: { + allowlist: readonly string[]; + orgGate: string; + orgLogins: readonly string[]; + username: string | undefined; +}): boolean { + const username = input.username?.trim().toLowerCase(); + if (username !== undefined && username.length > 0) { + if (input.allowlist.includes(username)) { + return true; + } + } + const orgGate = input.orgGate.trim().toLowerCase(); + if (orgGate.length === 0) { + return false; + } + return input.orgLogins.some( + (login) => login.trim().toLowerCase() === orgGate, + ); +} + +/** + * Evaluates the GitHub org gate for an OIDC callback. + * + * Allowlisted usernames skip the GitHub API. Public org membership is checked + * next without a GitHub token, so members who keep their membership public + * still get in when the OAuth App is not approved by the org. Private + * membership uses the Keycloak-stored GitHub token against + * `/user/memberships/orgs/{org}` (and `/user/orgs` as a fallback). + * + * A Keycloak session with no GitHub identity at all (seeded password users) + * is admitted: broker V1 returns 403 when the access token was never granted + * `broker/read-token`, which only happens for accounts that never went + * through the GitHub broker. That is the GitHub linkage check, not a JWT + * username guess. A session that *is* GitHub-linked but whose token cannot be + * read back (broker V1 404, "nothing is stored") fails closed rather than + * being admitted, since we cannot verify org membership for it. Any other + * lookup failure is also a denial. + */ +export async function evaluateGithubOrgGate( + input: GithubOrgGateInput, +): Promise { + const allowlist = parseUsernameAllowlist(input.allowlistRaw); + if ( + githubIdentityAllowed({ + allowlist, + orgGate: input.orgGate, + orgLogins: [], + username: input.username, + }) + ) { + return true; + } + + const fetchImpl = input.fetchImpl ?? globalThis.fetch; + const origin = input.githubApiOrigin.replace(/\/+$/u, ""); + try { + if ( + await isPublicOrgMember({ + fetchImpl, + githubApiOrigin: origin, + orgGate: input.orgGate, + username: input.username, + }) + ) { + return true; + } + const broker = await fetchBrokerGithubToken({ + accessToken: input.accessToken, + fetchImpl, + oidcIssuer: input.oidcIssuer, + }); + if (!broker.linked) { + return true; + } + const githubToken = broker.token; + if ( + await isActiveOrgMember({ + fetchImpl, + githubApiOrigin: origin, + githubToken, + orgGate: input.orgGate, + }) + ) { + return true; + } + const orgLogins = await fetchGithubOrgLogins({ + fetchImpl, + githubApiOrigin: origin, + githubToken, + }); + return githubIdentityAllowed({ + allowlist, + orgGate: input.orgGate, + orgLogins, + username: input.username, + }); + } catch (error) { + input.onLookupError?.(error); + return false; + } +} + +function githubApiHeaders(token?: string): Record { + const headers: Record = { + Accept: "application/vnd.github+json", + "User-Agent": "hypershell-web-console", + "X-GitHub-Api-Version": "2022-11-28", + }; + if (token !== undefined) { + headers.Authorization = `Bearer ${token}`; + } + return headers; +} + +/** + * Public membership does not require an org-approved OAuth App. GitHub + * returns 204 for public members and 404 for everyone else (including + * private members). + */ +async function isPublicOrgMember(input: { + fetchImpl: typeof fetch; + githubApiOrigin: string; + orgGate: string; + username: string | undefined; +}): Promise { + const username = input.username?.trim(); + const orgGate = input.orgGate.trim(); + if (username === undefined || username.length === 0 || orgGate.length === 0) { + return false; + } + const org = encodeURIComponent(orgGate); + const login = encodeURIComponent(username); + const response = await input.fetchImpl( + `${input.githubApiOrigin}/orgs/${org}/public_members/${login}`, + { + headers: githubApiHeaders(), + signal: AbortSignal.timeout(githubRequestTimeoutMs), + }, + ); + return response.status === 204; +} + +/** + * Private (and public) membership for the authenticated user. A 403 usually + * means the OAuth App is blocked by the org's third-party access policy, in + * which case `/user/orgs` also omits the org. + */ +async function isActiveOrgMember(input: { + fetchImpl: typeof fetch; + githubApiOrigin: string; + githubToken: string; + orgGate: string; +}): Promise { + const orgGate = input.orgGate.trim(); + if (orgGate.length === 0) { + return false; + } + const response = await input.fetchImpl( + `${input.githubApiOrigin}/user/memberships/orgs/${encodeURIComponent(orgGate)}`, + { + headers: githubApiHeaders(input.githubToken), + signal: AbortSignal.timeout(githubRequestTimeoutMs), + }, + ); + if (response.status === 404) { + return false; + } + if (response.status === 403) { + throw new Error( + `GitHub org membership failed with HTTP 403 (OAuth App may not be approved by ${orgGate})`, + ); + } + if (!response.ok) { + throw new Error( + `GitHub org membership failed with HTTP ${String(response.status)}`, + ); + } + const body: unknown = await response.json(); + return ( + typeof body === "object" && + body !== null && + "state" in body && + body.state === "active" + ); +} + +async function fetchGithubOrgLogins(input: { + fetchImpl: typeof fetch; + githubApiOrigin: string; + githubToken: string; +}): Promise { + const logins: string[] = []; + // Membership visibility depends on the GitHub IdP requesting `read:org` + // (deploy/base/keycloak GitHub identity provider defaultScope). Orgs that + // restrict third-party OAuth Apps omit themselves from this list. + const orgList = `${input.githubApiOrigin}/user/orgs?per_page=100`; + let nextUrl: string | undefined = orgList; + + for ( + let page = 0; + page < githubOrgPageLimit && nextUrl !== undefined; + page++ + ) { + const response = await input.fetchImpl(nextUrl, { + headers: githubApiHeaders(input.githubToken), + signal: AbortSignal.timeout(githubRequestTimeoutMs), + }); + if (!response.ok) { + throw new Error( + `GitHub org listing failed with HTTP ${String(response.status)}`, + ); + } + const body: unknown = await response.json(); + if (!Array.isArray(body)) { + throw new Error("GitHub org listing returned a non-array body"); + } + for (const entry of body) { + const login = githubOrgLogin(entry); + if (login !== undefined) { + logins.push(login); + } + } + nextUrl = nextLinkFrom(response.headers.get("link")); + } + + return logins; +} + +async function fetchBrokerGithubToken(input: { + accessToken: string; + fetchImpl: typeof fetch; + oidcIssuer: string; +}): Promise<{ linked: true; token: string } | { linked: false }> { + // storeToken + addReadTokenRoleOnCreate. Keycloak V1 retrieveToken: + // 403 if the access token has no broker/read-token, meaning the account + // never went through the GitHub broker (password users) - safe to treat + // as "not linked". 404 means the user *is* linked to the GitHub provider + // but Keycloak has nothing stored for it; that is a verifiable identity we + // failed to verify, so it must not be treated the same as "not linked". + // 200 is a GitHub login. + const issuer = input.oidcIssuer.replace(/\/+$/u, ""); + const response = await input.fetchImpl(`${issuer}/broker/github/token`, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${input.accessToken}`, + }, + signal: AbortSignal.timeout(githubRequestTimeoutMs), + }); + if (response.status === 403) { + return { linked: false }; + } + if (response.status === 404) { + throw new Error( + "Keycloak reports a linked GitHub identity with no stored broker token (HTTP 404)", + ); + } + if (!response.ok) { + throw new Error( + `Keycloak GitHub broker token failed with HTTP ${String(response.status)}`, + ); + } + const contentType = response.headers.get("content-type") ?? ""; + const body = await response.text(); + const token = readBrokerAccessToken(body, contentType); + if (token === undefined || token.length === 0) { + throw new Error( + "Keycloak GitHub broker token response had no access_token", + ); + } + return { linked: true, token }; +} + +function readBrokerAccessToken( + body: string, + contentType: string, +): string | undefined { + const trimmed = body.trim(); + if (trimmed.length === 0) { + return undefined; + } + // Keycloak 26 retrieveToken sets Content-Type application/json for whatever + // GitHub stored, including the default form-encoded access_token=... body. + if (contentType.includes("json") || trimmed.startsWith("{")) { + const fromJson = jsonAccessToken(trimmed); + if (fromJson !== undefined) { + return fromJson; + } + } + if ( + contentType.includes("application/x-www-form-urlencoded") || + trimmed.includes("access_token=") + ) { + return new URLSearchParams(trimmed).get("access_token") ?? undefined; + } + return undefined; +} + +function jsonAccessToken(body: string): string | undefined { + try { + const parsed: unknown = JSON.parse(body); + if ( + typeof parsed === "object" && + parsed !== null && + "access_token" in parsed && + typeof parsed.access_token === "string" + ) { + return parsed.access_token; + } + } catch { + return undefined; + } + return undefined; +} + +function githubOrgLogin(entry: unknown): string | undefined { + if (typeof entry !== "object" || entry === null || !("login" in entry)) { + return undefined; + } + return typeof entry.login === "string" ? entry.login : undefined; +} + +function nextLinkFrom(linkHeader: string | null): string | undefined { + if (linkHeader === null || linkHeader.length === 0) { + return undefined; + } + for (const part of linkHeader.split(",")) { + const match = /<([^>]+)>\s*;\s*rel="next"/u.exec(part); + const url = match?.[1]; + if (url !== undefined) { + return url; + } + } + return undefined; +} diff --git a/components/web-console/bff/test/auth.test.ts b/components/web-console/bff/test/auth.test.ts index 71d485e6e..a857e380c 100644 --- a/components/web-console/bff/test/auth.test.ts +++ b/components/web-console/bff/test/auth.test.ts @@ -51,6 +51,8 @@ const testSessionSecret = // --------------------------------------------------------------------------- interface OidcContext { + brokerStatus: number; + githubOrgs: string[]; nonce: string; port: number; } @@ -135,6 +137,38 @@ function createOidcServer(ctx: OidcContext): Server { return; } + if (url.pathname === "/broker/github/token") { + res.statusCode = ctx.brokerStatus; + if (res.statusCode !== 200) { + res.end("broker error"); + return; + } + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ access_token: "gho-test-token" })); + return; + } + + if (url.pathname === "/user/orgs") { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(ctx.githubOrgs.map((login) => ({ login })))); + return; + } + + if ( + url.pathname.startsWith("/orgs/") && + url.pathname.includes("/public_members/") + ) { + res.statusCode = 404; + res.end(); + return; + } + + if (url.pathname.startsWith("/user/memberships/orgs/")) { + res.statusCode = 404; + res.end(); + return; + } + res.statusCode = 404; res.end(); }); @@ -180,12 +214,29 @@ describe("OIDC configuration validation", () => { expect(config.sessionSecret).toBeInstanceOf(Buffer); expect(config.sessionSecret?.length).toBe(32); expect(config.sessionTtlSeconds).toBe(28_800); + expect(config.githubOrgGate).toBeUndefined(); + expect(config.githubApiOrigin).toBe("https://api.github.com"); }); it("accepts configuration without any OIDC settings", () => { const config = loadConfig({}); expect(config.oidcIssuer).toBeUndefined(); expect(config.sessionSecret).toBeUndefined(); + expect(config.githubOrgGate).toBeUndefined(); + }); + + it("treats a blank GITHUB_ORG_GATE as unset so Kind stays ungated", () => { + const config = loadConfig({ GITHUB_ORG_GATE: " " }); + expect(config.githubOrgGate).toBeUndefined(); + }); + + it("loads the GitHub org gate and allowlist", () => { + const config = loadConfig({ + GITHUB_ORG_GATE: "openshift-online", + GITHUB_USERNAME_ALLOWLIST: "alice,bob", + }); + expect(config.githubOrgGate).toBe("openshift-online"); + expect(config.githubUsernameAllowlist).toBe("alice,bob"); }); }); @@ -204,10 +255,18 @@ describe("web-console BFF with OIDC enabled", () => { method: string | undefined; url: string | undefined; }[]; - const oidcCtx: OidcContext = { nonce: "", port: 0 }; + const oidcCtx: OidcContext = { + brokerStatus: 200, + githubOrgs: ["openshift-online"], + nonce: "", + port: 0, + }; beforeEach(async () => { apiRequests = []; + oidcCtx.brokerStatus = 200; + oidcCtx.githubOrgs = ["openshift-online"]; + oidcCtx.nonce = ""; // --- mock upstream API --- apiServer = createServer( @@ -954,4 +1013,154 @@ describe("web-console BFF with OIDC enabled", () => { expect(response.statusCode).toBe(200); }); + + describe("GitHub org gate", () => { + let gatedApp: FastifyInstance; + + async function completeOidcLogin(target: FastifyInstance) { + const login = await target.inject({ method: "GET", url: "/auth/login" }); + if (typeof login.headers.location !== "string") { + throw new Error("Expected location header"); + } + const redirectUrl = new URL(login.headers.location); + const state = redirectUrl.searchParams.get("state"); + const nonce = redirectUrl.searchParams.get("nonce"); + if (!state || !nonce) { + throw new Error("Expected state and nonce in redirect URL"); + } + oidcCtx.nonce = nonce; + return target.inject({ + headers: { cookie: sessionCookie(login) }, + method: "GET", + url: `/auth/callback?code=test-code&state=${state}`, + }); + } + + function gatedConfig(overrides: Partial = {}): ServerConfig { + const apiAddr = apiServer.address(); + if (apiAddr === null || typeof apiAddr === "string") { + throw new Error("Expected TCP address for API server"); + } + return { + apiOrigin: `http://127.0.0.1:${String(apiAddr.port)}`, + apiTimeoutMs: 5000, + githubApiOrigin: `http://127.0.0.1:${String(oidcCtx.port)}`, + githubOrgGate: "openshift-online", + host: "127.0.0.1", + logLevel: "silent", + nodeEnv: "test", + oidcClientId: "test-client", + oidcIssuer: `http://127.0.0.1:${String(oidcCtx.port)}`, + oidcRedirectUri: "http://127.0.0.1:8080/auth/callback", + port: 8080, + prometheusQueryTimeoutMs: 10_000, + prometheusUrl: "http://127.0.0.1:9090", + sessionSecret: Buffer.from(testSessionSecret, "hex"), + sessionTtlSeconds: 28_800, + staticRoot, + ...overrides, + }; + } + + afterEach(async () => { + await gatedApp.close(); + }); + + it("creates a session for an organization member", async () => { + gatedApp = await buildApp(gatedConfig()); + const callback = await completeOidcLogin(gatedApp); + + expect(callback.statusCode).toBe(302); + expect(callback.headers.location).toBe("/"); + + const session = await gatedApp.inject({ + headers: { cookie: sessionCookie(callback) }, + method: "GET", + url: "/auth/session", + }); + expect(session.json()).toMatchObject({ authenticated: true }); + }); + + it("creates a session for an allowlisted username without calling GitHub", async () => { + oidcCtx.brokerStatus = 404; + oidcCtx.githubOrgs = []; + gatedApp = await buildApp( + gatedConfig({ githubUsernameAllowlist: "TestUser" }), + ); + const callback = await completeOidcLogin(gatedApp); + + expect(callback.statusCode).toBe(302); + expect(callback.headers.location).toBe("/"); + + const session = await gatedApp.inject({ + headers: { cookie: sessionCookie(callback) }, + method: "GET", + url: "/auth/session", + }); + expect(session.json()).toMatchObject({ authenticated: true }); + }); + + it("redirects non-members to /auth/denied without a session", async () => { + oidcCtx.githubOrgs = ["acme"]; + gatedApp = await buildApp(gatedConfig()); + const callback = await completeOidcLogin(gatedApp); + + expect(callback.statusCode).toBe(302); + expect(callback.headers.location).toBe("/auth/denied"); + + const session = await gatedApp.inject({ + headers: { cookie: sessionCookie(callback) }, + method: "GET", + url: "/auth/session", + }); + expect(session.json()).toEqual({ authenticated: false }); + + const denied = await gatedApp.inject({ + method: "GET", + url: "/auth/denied", + }); + expect(denied.statusCode).toBe(403); + expect(denied.headers["content-type"]).toContain("text/html"); + expect(denied.body).toContain("Access denied"); + expect(denied.body).toContain('class="card"'); + expect(denied.headers["content-security-policy"]).toMatch( + /style-src[^;]*'sha256-/u, + ); + + const api = await gatedApp.inject({ + headers: { cookie: sessionCookie(callback) }, + method: "GET", + url: "/api/hypershell/v1/gateways", + }); + expect(api.statusCode).toBe(401); + expect(api.json()).toMatchObject({ error: "reauth_required" }); + expect(apiRequests).toHaveLength(0); + }); + + it("denies login when the GitHub broker token cannot be read", async () => { + oidcCtx.brokerStatus = 401; + gatedApp = await buildApp(gatedConfig()); + const callback = await completeOidcLogin(gatedApp); + + expect(callback.statusCode).toBe(302); + expect(callback.headers.location).toBe("/auth/denied"); + }); + + it("creates a session when Keycloak has no GitHub broker identity", async () => { + oidcCtx.brokerStatus = 403; + oidcCtx.githubOrgs = []; + gatedApp = await buildApp(gatedConfig()); + const callback = await completeOidcLogin(gatedApp); + + expect(callback.statusCode).toBe(302); + expect(callback.headers.location).toBe("/"); + + const session = await gatedApp.inject({ + headers: { cookie: sessionCookie(callback) }, + method: "GET", + url: "/auth/session", + }); + expect(session.json()).toMatchObject({ authenticated: true }); + }); + }); }); diff --git a/components/web-console/bff/test/config.test.ts b/components/web-console/bff/test/config.test.ts index b3abf5cd3..1a5584c5e 100644 --- a/components/web-console/bff/test/config.test.ts +++ b/components/web-console/bff/test/config.test.ts @@ -108,6 +108,34 @@ describe("loadConfig", () => { }), ).toThrow(/OTEL_TRACES_SAMPLE_RATIO/u); }); + + it("defaults GitHub org-gate settings to unset", () => { + const config = loadConfig({ STATIC_ROOT: "./public" }); + + expect(config.githubOrgGate).toBeUndefined(); + expect(config.githubUsernameAllowlist).toBeUndefined(); + expect(config.githubApiOrigin).toBe("https://api.github.com"); + }); + + it("loads the GitHub org gate and allowlist from the environment", () => { + const config = loadConfig({ + GITHUB_ORG_GATE: "openshift-online", + GITHUB_USERNAME_ALLOWLIST: "alice,bob", + STATIC_ROOT: "./public", + }); + + expect(config.githubOrgGate).toBe("openshift-online"); + expect(config.githubUsernameAllowlist).toBe("alice,bob"); + }); + + it("treats a blank GitHub org gate as unset", () => { + const config = loadConfig({ + GITHUB_ORG_GATE: " ", + STATIC_ROOT: "./public", + }); + + expect(config.githubOrgGate).toBeUndefined(); + }); }); describe("browserRuntimeConfig", () => { @@ -133,6 +161,7 @@ describe("browserRuntimeConfig", () => { it("exposes no server-only configuration to the browser", () => { const config = loadConfig({ + GITHUB_ORG_GATE: "openshift-online", OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector.example.test:4318", SESSION_SECRET: "a".repeat(64), STATIC_ROOT: "./public", @@ -141,6 +170,7 @@ describe("browserRuntimeConfig", () => { const serialized = JSON.stringify(browserRuntimeConfig(config)); expect(serialized).not.toContain("collector.example.test"); expect(serialized).not.toContain("a".repeat(64)); + expect(serialized).not.toContain("openshift-online"); expect(Object.keys(browserRuntimeConfig(config))).toEqual(["tracing"]); }); }); diff --git a/components/web-console/bff/test/github-org-gate.test.ts b/components/web-console/bff/test/github-org-gate.test.ts new file mode 100644 index 000000000..b29b0db6e --- /dev/null +++ b/components/web-console/bff/test/github-org-gate.test.ts @@ -0,0 +1,417 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + evaluateGithubOrgGate, + githubIdentityAllowed, + parseUsernameAllowlist, +} from "../src/github-org-gate.js"; + +describe("parseUsernameAllowlist", () => { + it("splits, trims, and lowercases comma-separated usernames", () => { + expect(parseUsernameAllowlist(" Alice,Bob , ,CAROL ")).toEqual([ + "alice", + "bob", + "carol", + ]); + }); + + it("returns an empty list when the value is missing or blank", () => { + expect(parseUsernameAllowlist(undefined)).toEqual([]); + expect(parseUsernameAllowlist("")).toEqual([]); + expect(parseUsernameAllowlist(" , ")).toEqual([]); + }); +}); + +describe("githubIdentityAllowed", () => { + it("admits an organization member", () => { + expect( + githubIdentityAllowed({ + allowlist: [], + orgGate: "openshift-online", + orgLogins: ["kubernetes", "openshift-online"], + username: "alice", + }), + ).toBe(true); + }); + + it("compares organization membership case-insensitively", () => { + expect( + githubIdentityAllowed({ + allowlist: [], + orgGate: "OpenShift-Online", + orgLogins: ["OpenShift-Online"], + username: "alice", + }), + ).toBe(true); + }); + + it("admits an allowlisted username who is not an org member", () => { + expect( + githubIdentityAllowed({ + allowlist: ["outside-contributor"], + orgGate: "openshift-online", + orgLogins: ["acme"], + username: "Outside-Contributor", + }), + ).toBe(true); + }); + + it("denies a user who is neither an org member nor allowlisted", () => { + expect( + githubIdentityAllowed({ + allowlist: ["someone-else"], + orgGate: "openshift-online", + orgLogins: ["acme"], + username: "alice", + }), + ).toBe(false); + }); + + it("denies when the org list is empty and the username is not allowlisted", () => { + expect( + githubIdentityAllowed({ + allowlist: [], + orgGate: "openshift-online", + orgLogins: [], + username: "alice", + }), + ).toBe(false); + }); +}); + +function hrefOf(input: Parameters[0]): string { + if (typeof input === "string") { + return input; + } + if (input instanceof URL) { + return input.href; + } + return input.url; +} + +describe("evaluateGithubOrgGate", () => { + const baseInput = { + accessToken: "kc-access-token", + githubApiOrigin: "https://api.github.com", + oidcIssuer: "https://sso.example.test/realms/hypershell", + orgGate: "openshift-online", + username: "alice", + }; + + it("skips GitHub when the username is allowlisted", async () => { + const fetchImpl = vi.fn(); + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "alice", + fetchImpl, + }), + ).resolves.toBe(true); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("admits a public org member without reading the broker token", async () => { + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.includes("/public_members/")) { + return Promise.resolve(new Response(null, { status: 204 })); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(true); + expect(fetchImpl).toHaveBeenCalledTimes(1); + const firstCall = fetchImpl.mock.calls[0]?.[0]; + expect(firstCall).toBeDefined(); + if (firstCall === undefined) { + throw new Error("expected a public-members fetch"); + } + expect(hrefOf(firstCall)).toContain( + "/orgs/openshift-online/public_members/alice", + ); + }); + + it("admits a private member via /user/memberships when /user/orgs omits the org", async () => { + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.includes("/public_members/")) { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response(JSON.stringify({ access_token: "gho-test" }), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + } + if (href.includes("/user/memberships/orgs/")) { + return Promise.resolve( + new Response(JSON.stringify({ state: "active" }), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + } + if (href.includes("/user/orgs")) { + return Promise.resolve( + new Response(JSON.stringify([]), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(true); + expect( + fetchImpl.mock.calls.some((call) => + hrefOf(call[0]).includes("/user/orgs"), + ), + ).toBe(false); + }); + + it("denies when GitHub returns 403 because the OAuth App is not org-approved", async () => { + const onLookupError = vi.fn(); + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.includes("/public_members/")) { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response(JSON.stringify({ access_token: "gho-test" }), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + } + if (href.includes("/user/memberships/orgs/")) { + return Promise.resolve( + new Response("OAuth App access restrictions", { status: 403 }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + onLookupError, + }), + ).resolves.toBe(false); + expect(onLookupError).toHaveBeenCalledTimes(1); + expect(String(onLookupError.mock.calls[0]?.[0])).toMatch( + /OAuth App may not be approved by openshift-online/u, + ); + }); + + it("admits an org member after reading the brokered GitHub token", async () => { + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response(JSON.stringify({ access_token: "gho-test" }), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + } + if (href.includes("/user/orgs")) { + return Promise.resolve( + new Response( + JSON.stringify([{ login: "openshift-online" }, { login: "other" }]), + { + headers: { "content-type": "application/json" }, + status: 200, + }, + ), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(true); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + + it("admits a login with no GitHub broker identity (password users)", async () => { + const onLookupError = vi.fn(); + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.includes("/public_members/")) { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response("Client not authorized to retrieve tokens", { + status: 403, + }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + onLookupError, + }), + ).resolves.toBe(true); + expect(onLookupError).not.toHaveBeenCalled(); + }); + + it("denies a login when Keycloak reports a linked identity with no stored token", async () => { + const onLookupError = vi.fn(); + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.includes("/public_members/")) { + return Promise.resolve(new Response("not found", { status: 404 })); + } + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response("No token stored", { status: 404 }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + onLookupError, + }), + ).resolves.toBe(false); + expect(onLookupError).toHaveBeenCalledTimes(1); + expect(String(onLookupError.mock.calls[0]?.[0])).toMatch( + /linked GitHub identity with no stored broker token/u, + ); + }); + + it("denies when GitHub org lookup fails", async () => { + const fetchImpl = vi.fn(() => + Promise.resolve(new Response("nope", { status: 401 })), + ); + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(false); + }); + + it("denies a non-member when the org list does not include the gate", async () => { + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response("access_token=gho-test&token_type=bearer", { + headers: { "content-type": "application/x-www-form-urlencoded" }, + status: 200, + }), + ); + } + return Promise.resolve( + new Response(JSON.stringify([{ login: "acme" }]), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(false); + }); + + it("admits an org member when Keycloak labels a form-encoded broker token as JSON", async () => { + // Keycloak 26 retrieveToken always sets Content-Type application/json even + // when GitHub stored access_token=...&token_type=bearer (the default + // GitHub token response unless githubJsonFormat is enabled). + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response( + "access_token=gho-test&token_type=bearer&scope=read%3Aorg", + { + headers: { "content-type": "application/json" }, + status: 200, + }, + ), + ); + } + if (href.includes("/user/orgs")) { + return Promise.resolve( + new Response(JSON.stringify([{ login: "openshift-online" }]), { + headers: { "content-type": "application/json" }, + status: 200, + }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(true); + }); + + it("denies when the broker token body is neither JSON nor form-encoded", async () => { + const fetchImpl = vi.fn((input: Parameters[0]) => { + const href = hrefOf(input); + if (href.endsWith("/broker/github/token")) { + return Promise.resolve( + new Response("gho-not-a-structured-token", { + headers: { "content-type": "text/plain" }, + status: 200, + }), + ); + } + return Promise.resolve(new Response("not found", { status: 404 })); + }); + + await expect( + evaluateGithubOrgGate({ + ...baseInput, + allowlistRaw: "", + fetchImpl, + }), + ).resolves.toBe(false); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); +}); diff --git a/deploy/base/keycloak/keycloak.yaml b/deploy/base/keycloak/keycloak.yaml index 2a0308f7a..544f03444 100644 --- a/deploy/base/keycloak/keycloak.yaml +++ b/deploy/base/keycloak/keycloak.yaml @@ -14,12 +14,101 @@ spec: spec: securityContext: runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + initContainers: + # Renders GitHub IdP / e2e-client values and OpenShift console redirect + # URIs into the realm JSON. Keycloak --import-realm does not substitute + # placeholders (upstream keycloak#20199), and start-dev's ephemeral H2 + # store drops any admin-API redirect URI patch on pod recycle. + - name: render-realm-config + image: registry.access.redhat.com/hi/python:3.13-builder@sha256:75f0b15a73e9510e97dc3c3613a8c17794a0efbfc42ecfb4203e419450163763 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + env: + - name: PR_ENV_GITHUB_IDP_ENABLED + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: idp-enabled + optional: true + - name: PR_ENV_GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: client-id + optional: true + - name: PR_ENV_GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: client-secret + optional: true + - name: PR_ENV_GITHUB_ORG + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: org + optional: true + - name: PR_ENV_GITHUB_ALLOWLIST + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: allowlist + optional: true + - name: HYPERSHELL_E2E_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: e2e-client-secret + optional: true + - name: HYPERSHELL_E2E_CLIENT_ENABLED + valueFrom: + secretKeyRef: + name: hypershell-github-oauth + key: e2e-client-enabled + optional: true + # Set by make openshift-up on this init container from the + # web-console Route host. Unset on Kind, so the renderer keeps the + # localhost frontend redirect URIs. Must not live only on the + # keycloak container: start-dev --import-realm reads this render. + - name: HYPERSHELL_CONSOLE_HOST + value: "" + command: + - python3 + - /scripts/render-realm-config.py + - /config/hypershell-realm.json + - /rendered/hypershell-realm.json + volumeMounts: + - name: realm-config-source + mountPath: /config + readOnly: true + - name: realm-renderer + mountPath: /scripts + readOnly: true + - name: realm-config + mountPath: /rendered containers: - name: keycloak image: quay.io/keycloak/keycloak:26.7.2@sha256:831330513f55695572286e521f94fcd3c7e285250ed5b848090265a33192f669 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault capabilities: drop: ["ALL"] args: @@ -44,6 +133,13 @@ spec: value: "https://keycloak.hypershell.localhost" - name: KC_PROXY_HEADERS value: "xforwarded" + # Legacy token-exchange (requested_subject impersonation) plus FGAP + # v1. Keycloak 26 standard token-exchange rejects that parameter, + # and FGAP v2 (the 26.2+ default) has no token-exchange permission. + # The e2e suite impersonates seeded users onto gateway and frontend + # audiences, which needs both features. + - name: KC_FEATURES + value: "token-exchange,admin-fine-grained-authz:v1" ports: - containerPort: 8080 name: http @@ -92,9 +188,15 @@ spec: mountPath: /opt/keycloak/themes/hypershell/login/messages/messages_en.properties subPath: messages_en.properties volumes: - - name: realm-config + - name: realm-config-source configMap: name: keycloak-realm + - name: realm-renderer + configMap: + name: keycloak-realm-renderer + defaultMode: 0444 + - name: realm-config + emptyDir: {} - name: hypershell-theme configMap: name: keycloak-hypershell-theme @@ -114,6 +216,24 @@ spec: port: 8080 targetPort: 8080 --- +# hypershell-e2e client: confidential CI client for the OpenShift pull-request +# e2e suite (ephemeral-pr-environments.spec.md). The render-realm-config init +# container omits this client, its service-account user, and the impersonation +# clientScopeMapping unless HYPERSHELL_E2E_CLIENT_ENABLED=true, so Kind, local, +# and hub imported realms do not contain the privileged identity. A disabled +# leftover in an already-imported realm is still skipped by +# EnsureE2ETokenExchange (enabled==true, not mere presence). PR environments +# set e2e-client-enabled=true and a random secret via hypershell-github-oauth. +# The service account holds platform:admin + gateway:creator so +# E2E_OIDC_GRANT=client_credentials can obtain admin tokens without a GitHub +# login. Standard token exchange stays off; Area 4 uses FGAP v1 token-exchange +# granted at gateway reconcile. Distinct from hypershell-provisioner +# (manage-clients / manage-users / manage-authorization, the last needed to +# manage the hypershell-e2e-token-exchange authorization policy on +# realm-management; see keycloak/client.go EnsureE2ETokenExchange). +# The "description" +# field below is kept short because Keycloak's CLIENT.DESCRIPTION column is +# VARCHAR(255); exceeding that fails the whole realm import at boot. apiVersion: v1 kind: ConfigMap metadata: @@ -135,7 +255,7 @@ data: "fullScopeAllowed": true, "redirectUris": ["https://console.hypershell.localhost/*", "https://console.hypershell.localhost:*"], "webOrigins": ["+"], - "defaultClientScopes": ["openid", "email", "profile"], + "defaultClientScopes": ["openid", "email", "profile", "roles"], "protocolMappers": [ { "name": "audience", @@ -268,6 +388,48 @@ data: "directAccessGrantsEnabled": false, "secret": "provisioner-secret" }, + { + "clientId": "hypershell-e2e", + "description": "Confidential CI client for the OpenShift e2e suite (ephemeral-pr-environments.spec.md)", + "enabled": "${HYPERSHELL_E2E_CLIENT_ENABLED:false}", + "publicClient": false, + "serviceAccountsEnabled": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": false, + "secret": "${HYPERSHELL_E2E_CLIENT_SECRET:}", + "attributes": { + "standard.token.exchange.enabled": "false" + }, + "defaultClientScopes": ["profile", "roles"], + "protocolMappers": [ + { + "name": "audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "hypershell-frontend", + "id.token.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + }, + { + "name": "preferred-username", + "protocol": "openid-connect", + "protocolMapper": "oidc-hardcoded-claim-mapper", + "consentRequired": false, + "config": { + "claim.name": "preferred_username", + "claim.value": "admin", + "jsonType.label": "String", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, { "clientId": "hypershell-control-plane", "enabled": true, @@ -568,13 +730,62 @@ data: "enabled": true, "serviceAccountClientId": "hypershell-provisioner", "clientRoles": { - "realm-management": ["manage-clients", "manage-users"] + "realm-management": ["manage-clients", "manage-users", "manage-authorization"] + } + }, + { + "username": "service-account-hypershell-e2e", + "enabled": false, + "serviceAccountClientId": "hypershell-e2e", + "realmRoles": ["platform:admin", "gateway:creator"], + "clientRoles": { + "realm-management": ["impersonation"] } } ], "clientScopeMappings": { "hypershell-provisioner": [ - { "client": "realm-management", "roles": ["manage-clients", "manage-users"] } + { "client": "realm-management", "roles": ["manage-clients", "manage-users", "manage-authorization"] } + ], + "hypershell-e2e": [ + { "client": "realm-management", "roles": ["impersonation"] } ] - } + }, + "attributes": { + "github.org.gate": "${PR_ENV_GITHUB_ORG:openshift-online}", + "github.username.allowlist": "${PR_ENV_GITHUB_ALLOWLIST:}" + }, + "identityProviders": [ + { + "alias": "github", + "providerId": "github", + "enabled": "${PR_ENV_GITHUB_IDP_ENABLED:false}", + "trustEmail": true, + "storeToken": true, + "addReadTokenRoleOnCreate": true, + "firstBrokerLoginFlowAlias": "first broker login", + "config": { + "clientId": "${PR_ENV_GITHUB_CLIENT_ID:}", + "clientSecret": "${PR_ENV_GITHUB_CLIENT_SECRET:}", + "defaultScope": "read:org user:email", + "githubJsonFormat": "true", + "caseSensitiveOriginalUsername": "false", + "syncMode": "FORCE" + } + } + ], + "identityProviderMappers": [ + { + "name": "github-grant-platform-admin", + "identityProviderAlias": "github", + "identityProviderMapper": "oidc-hardcoded-role-idp-mapper", + "config": { "role": "platform:admin", "syncMode": "FORCE" } + }, + { + "name": "github-grant-gateway-creator", + "identityProviderAlias": "github", + "identityProviderMapper": "oidc-hardcoded-role-idp-mapper", + "config": { "role": "gateway:creator", "syncMode": "FORCE" } + } + ] } diff --git a/deploy/base/keycloak/kustomization.yaml b/deploy/base/keycloak/kustomization.yaml index 2c049ecce..20bcd93f9 100644 --- a/deploy/base/keycloak/kustomization.yaml +++ b/deploy/base/keycloak/kustomization.yaml @@ -8,3 +8,9 @@ resources: # standalone (e.g. remote-ref'd by an operator-managed Keycloak). Pulling it # here keeps this base's rendered output unchanged. - theme +configMapGenerator: + - name: keycloak-realm-renderer + options: + disableNameSuffixHash: true + files: + - render-realm-config.py diff --git a/deploy/base/keycloak/render-realm-config.py b/deploy/base/keycloak/render-realm-config.py new file mode 100755 index 000000000..d848bf70d --- /dev/null +++ b/deploy/base/keycloak/render-realm-config.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Render the HyperShell Keycloak realm for the current environment. + +Keycloak --import-realm does not substitute ${VAR:default} placeholders +(upstream keycloak#20199). This script is the init-container renderer: it +loads the ConfigMap JSON, applies env-gated GitHub IdP / e2e-client values +(omitting the privileged hypershell-e2e identity unless it is enabled), +and (on OpenShift) replaces hypershell-frontend redirect URIs with the +web-console Route origin so they survive Keycloak pod restarts. start-dev +uses an ephemeral H2 store, so any admin-API mutation of redirect URIs is +lost on recycle; baking them into the imported JSON is the durable path. +""" + +from __future__ import annotations + +import json +import os +import sys +from typing import Any + + +def render_realm(realm: dict[str, Any], environ: dict[str, str] | None = None) -> dict[str, Any]: + env = os.environ if environ is None else environ + idp_enabled = env.get("PR_ENV_GITHUB_IDP_ENABLED", "false").strip().lower() == "true" + e2e_enabled = env.get("HYPERSHELL_E2E_CLIENT_ENABLED", "false").strip().lower() == "true" + console_host = env.get("HYPERSHELL_CONSOLE_HOST", "").strip() + + attributes = realm.setdefault("attributes", {}) + attributes["github.org.gate"] = env.get("PR_ENV_GITHUB_ORG", "openshift-online") + attributes["github.username.allowlist"] = env.get("PR_ENV_GITHUB_ALLOWLIST", "") + + for idp in realm.get("identityProviders") or []: + if idp.get("alias") != "github": + continue + idp["enabled"] = idp_enabled + config = idp.setdefault("config", {}) + config["clientId"] = env.get("PR_ENV_GITHUB_CLIENT_ID", "") + config["clientSecret"] = env.get("PR_ENV_GITHUB_CLIENT_SECRET", "") + + clients: list[dict[str, Any]] = [] + for client in realm.get("clients") or []: + client_id = client.get("clientId") + if client_id == "hypershell-e2e": + # Omit the privileged CI client from Kind, local OpenShift, and + # hub imports. A disabled-but-present client still has a UUID, and + # the control plane must not grant token-exchange from it. + if not e2e_enabled: + continue + client["enabled"] = True + client["secret"] = env.get("HYPERSHELL_E2E_CLIENT_SECRET", "") + if client_id == "hypershell-frontend" and console_host: + # Exact console origin only. Wildcard redirect URIs are forbidden + # (oidc-integration.spec.md / openshift-development.spec.md). + client["redirectUris"] = [ + f"https://{console_host}/auth/callback", + f"https://{console_host}", + ] + clients.append(client) + realm["clients"] = clients + + e2e_user = "service-account-hypershell-e2e" + users: list[dict[str, Any]] = [] + for user in realm.get("users") or []: + is_e2e_sa = ( + user.get("username") == e2e_user + or user.get("serviceAccountClientId") == "hypershell-e2e" + ) + if is_e2e_sa and not e2e_enabled: + continue + if is_e2e_sa: + user["enabled"] = True + users.append(user) + realm["users"] = users + + mappings = realm.get("clientScopeMappings") or {} + if e2e_enabled: + realm["clientScopeMappings"] = mappings + else: + mappings.pop("hypershell-e2e", None) + realm["clientScopeMappings"] = mappings + + return realm + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + src = args[0] if len(args) > 0 else "/config/hypershell-realm.json" + dst = args[1] if len(args) > 1 else "/rendered/hypershell-realm.json" + with open(src, encoding="utf-8") as handle: + realm = json.load(handle) + if not isinstance(realm, dict): + raise SystemExit(f"realm JSON root must be an object, got {type(realm).__name__}") + render_realm(realm) + with open(dst, "w", encoding="utf-8") as handle: + json.dump(realm, handle, indent=2) + handle.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/base/keycloak/render-realm-config_test.sh b/deploy/base/keycloak/render-realm-config_test.sh new file mode 100755 index 000000000..3c042366c --- /dev/null +++ b/deploy/base/keycloak/render-realm-config_test.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Unit tests for deploy/base/keycloak/render-realm-config.py. +# Reproduces the OpenShift "Invalid parameter: redirect_uri" failure: a Keycloak +# pod recycle re-imports the realm, so console redirect URIs must be in the +# rendered JSON (not only patched via the admin API after boot). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RENDER="${SCRIPT_DIR}/render-realm-config.py" +REALM_YAML="${SCRIPT_DIR}/keycloak.yaml" + +PASS=0 +FAIL=0 + +assert_eq() { + local want="$1" got="$2" label="$3" + if [[ "${want}" == "${got}" ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + printf 'FAIL: %s (want=%q got=%q)\n' "${label}" "${want}" "${got}" + fi +} + +extract_realm() { + python3 - "$REALM_YAML" <<'PY' +import sys +from pathlib import Path +text = Path(sys.argv[1]).read_text() +marker = "hypershell-realm.json: |" +idx = text.index(marker) + len(marker) +body = [] +for line in text[idx:].splitlines(): + if line.startswith(" "): + body.append(line[4:]) + continue + if line.strip() == "": + body.append("") + continue + break +print("\n".join(body).strip()) +PY +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT +extract_realm >"${WORKDIR}/hypershell-realm.json" +python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "${WORKDIR}/hypershell-realm.json" + +# --- Kind / no console host: localhost redirect URIs stay, GitHub IdP off --- +env -i PATH="${PATH}" python3 "${RENDER}" \ + "${WORKDIR}/hypershell-realm.json" "${WORKDIR}/kind.json" +kind_redirects="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); client=next(c for c in realm["clients"] if c["clientId"]=="hypershell-frontend"); print(json.dumps(client["redirectUris"]))' "${WORKDIR}/kind.json")" +assert_eq '["https://console.hypershell.localhost/*", "https://console.hypershell.localhost:*"]' \ + "${kind_redirects}" "Kind keeps localhost frontend redirect URIs" +kind_idp="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print(json.dumps(next(i for i in realm["identityProviders"] if i["alias"]=="github")["enabled"]))' "${WORKDIR}/kind.json")" +assert_eq 'false' "${kind_idp}" "Kind leaves GitHub IdP disabled as JSON boolean" +kind_e2e="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print(any(c.get("clientId")=="hypershell-e2e" for c in realm["clients"]))' "${WORKDIR}/kind.json")" +assert_eq 'False' "${kind_e2e}" "Kind omits hypershell-e2e from the imported realm" +kind_e2e_user="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print(any(u.get("serviceAccountClientId")=="hypershell-e2e" or u.get("username")=="service-account-hypershell-e2e" for u in realm["users"]))' "${WORKDIR}/kind.json")" +assert_eq 'False' "${kind_e2e_user}" "Kind omits service-account-hypershell-e2e from the imported realm" +kind_e2e_mapping="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print("hypershell-e2e" in (realm.get("clientScopeMappings") or {}))' "${WORKDIR}/kind.json")" +assert_eq 'False' "${kind_e2e_mapping}" "Kind omits hypershell-e2e impersonation clientScopeMappings" + +# --- OpenShift console host: exact callback URIs, no localhost wildcards --- +env -i PATH="${PATH}" \ + HYPERSHELL_CONSOLE_HOST='web-console-hypershell-ci-pr-267.apps.rosa.example.com' \ + PR_ENV_GITHUB_IDP_ENABLED=true \ + PR_ENV_GITHUB_CLIENT_ID='Iv1.example' \ + PR_ENV_GITHUB_CLIENT_SECRET='s3cr3t' \ + PR_ENV_GITHUB_ORG='openshift-online' \ + HYPERSHELL_E2E_CLIENT_ENABLED=true \ + HYPERSHELL_E2E_CLIENT_SECRET='e2e-secret' \ + python3 "${RENDER}" "${WORKDIR}/hypershell-realm.json" "${WORKDIR}/pr.json" + +pr_redirects="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); client=next(c for c in realm["clients"] if c["clientId"]=="hypershell-frontend"); print(json.dumps(client["redirectUris"]))' "${WORKDIR}/pr.json")" +assert_eq '["https://web-console-hypershell-ci-pr-267.apps.rosa.example.com/auth/callback", "https://web-console-hypershell-ci-pr-267.apps.rosa.example.com"]' \ + "${pr_redirects}" "PR env frontend redirect URIs match the console Route" +if printf '%s' "${pr_redirects}" | grep -q localhost; then + FAIL=$((FAIL + 1)) + echo 'FAIL: PR env frontend redirect URIs still include localhost (Keycloak would reject the BFF redirect_uri)' +else + PASS=$((PASS + 1)) +fi +pr_idp="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); idp=next(i for i in realm["identityProviders"] if i["alias"]=="github"); print(json.dumps({"enabled": idp["enabled"], "clientId": idp["config"]["clientId"]}))' "${WORKDIR}/pr.json")" +assert_eq '{"enabled": true, "clientId": "Iv1.example"}' "${pr_idp}" "PR env GitHub IdP enabled with OAuth client id" +pr_broker="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); idp=next(i for i in realm["identityProviders"] if i["alias"]=="github"); print(json.dumps({"storeToken": idp.get("storeToken"), "addReadTokenRoleOnCreate": idp.get("addReadTokenRoleOnCreate"), "githubJsonFormat": idp["config"].get("githubJsonFormat")}))' "${WORKDIR}/pr.json")" +assert_eq '{"storeToken": true, "addReadTokenRoleOnCreate": true, "githubJsonFormat": "true"}' \ + "${pr_broker}" "GitHub IdP stores a JSON GitHub token and grants broker read-token on first login" +pr_frontend_scopes="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); client=next(c for c in realm["clients"] if c["clientId"]=="hypershell-frontend"); print(",".join(client["defaultClientScopes"]))' "${WORKDIR}/pr.json")" +assert_eq 'openid,email,profile,roles' \ + "${pr_frontend_scopes}" "Frontend access tokens include client roles so broker.read-token can be presented" +pr_mappers="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print(",".join(sorted({m["identityProviderMapper"] for m in realm["identityProviderMappers"]})))' "${WORKDIR}/pr.json")" +assert_eq 'oidc-hardcoded-role-idp-mapper' \ + "${pr_mappers}" "GitHub IdP hardcoded-role mapper uses the Keycloak 26 provider id" +pr_e2e="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print(json.dumps(next(c for c in realm["clients"] if c["clientId"]=="hypershell-e2e")["enabled"]))' "${WORKDIR}/pr.json")" +assert_eq 'true' "${pr_e2e}" "PR env imports hypershell-e2e enabled" +pr_e2e_user="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print(json.dumps(next(u["enabled"] for u in realm["users"] if u.get("serviceAccountClientId")=="hypershell-e2e")))' "${WORKDIR}/pr.json")" +assert_eq 'true' "${pr_e2e_user}" "PR env enables service-account-hypershell-e2e" +pr_e2e_mapping="$(python3 -c 'import json,sys; realm=json.load(open(sys.argv[1])); print("impersonation" in (realm.get("clientScopeMappings") or {}).get("hypershell-e2e", [{}])[0].get("roles", []))' "${WORKDIR}/pr.json")" +assert_eq 'True' "${pr_e2e_mapping}" "PR env keeps hypershell-e2e impersonation clientScopeMappings" + +if grep -q 'value: "token-exchange,admin-fine-grained-authz:v1"' "${REALM_YAML}"; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: Keycloak Deployment must enable token-exchange and admin-fine-grained-authz:v1' +fi + +printf 'render-realm-config tests: %d passed, %d failed\n' "$PASS" "$FAIL" +[[ "${FAIL}" -eq 0 ]] diff --git a/deploy/base/keycloak/theme/login.css b/deploy/base/keycloak/theme/login.css index f87026294..edbc43e49 100644 --- a/deploy/base/keycloak/theme/login.css +++ b/deploy/base/keycloak/theme/login.css @@ -225,7 +225,17 @@ h1.pf-v5-c-title.pf-m-3xl, transition: color 0.2s ease, border-color 0.2s ease; } +/* + * The control button's ::after border is composed from four separate + * top/right/bottom/left color variables. We only drove the bottom one (for + * the underline-style hover highlight), leaving top/right at PatternFly's + * own default (--pf-v5-global--BorderColor--300, #f0f0f0) -- nearly + * invisible against the white card, so the button looked borderless on top + * next to the password input's visibly bordered #e0e0e0 edge. + */ .pf-v5-c-button.pf-m-control::after { + border-top-color: #e0e0e0 !important; + border-right-color: #e0e0e0 !important; border-bottom-color: #e0e0e0 !important; transition: border-bottom-color 0.2s ease; } @@ -280,6 +290,93 @@ h1.pf-v5-c-title.pf-m-3xl, background-color: transparent; } +/* + * Social login buttons (e.g. "Sign in with GitHub"). Keycloak ships these as + * a full-width pf-m-secondary button (blue outline, blue text) with a raw + * icon that has no width/height attributes -- browsers fall back to a + * 300x150 default replaced-element size, ballooning the whole button. Reskin + * to match PatternFly's tertiary button (transparent fill, pill border, icon + * and text centered together) and size the icon explicitly. + * + * PatternFly's secondary/tertiary variants don't draw their border on the + * button box (box border-width is 0) -- it's painted by a ::after + * pseudo-element with its own --pf-v5-c-button--after--BorderRadius, which + * defaults to a small radius. Overriding the box's border-radius alone + * rounds an invisible box and leaves the visible border square, so the + * pseudo-element's radius and border color both need to be driven here. + */ +#kc-social-providers .pf-v5-c-button.pf-m-secondary { + --pf-v5-c-button--after--BorderRadius: 50rem; + --pf-v5-c-button--m-secondary--after--BorderColor: #0066cc; + --pf-v5-c-button--m-secondary--hover--after--BorderColor: #004d99; + --pf-v5-c-button--m-secondary--focus--after--BorderColor: #004d99; + --pf-v5-c-button--m-secondary--active--after--BorderColor: #004d99; + background-color: transparent !important; + color: #0066cc !important; + border-radius: 50rem !important; + font-family: 'Red Hat Text', system-ui, -apple-system, sans-serif !important; + justify-content: center !important; + gap: 0.5rem; + transition: color 0.2s ease; +} + +#kc-social-providers .pf-v5-c-button.pf-m-secondary:hover, +#kc-social-providers .pf-v5-c-button.pf-m-secondary:focus, +#kc-social-providers .pf-v5-c-button.pf-m-secondary:active { + color: #004d99 !important; + background-color: transparent !important; + box-shadow: none !important; + outline: none !important; +} + +/* The provider name ships with the pf-v5-u-m-auto utility (margin: auto + !important), which absorbs the button's flex free space on its own and + pins the icon to the far left regardless of justify-content. Zero it so + the icon and name sit next to each other as a single centered group. */ +#kc-social-providers .pf-v5-c-button.pf-m-secondary > span { + margin: 0 !important; +} + +#kc-social-providers svg { + width: 1.25rem !important; + height: 1.25rem !important; + flex-shrink: 0; +} + +/* The provider list reuses the .pf-v5-c-login__main-body class for its + horizontal edge-to-edge padding, but that class also carries the same + large PaddingBottom as the real card body wrapping it -- doubling up and + leaving a large gap under the button before the card ends. */ +#kc-social-providers .pf-v5-c-login__main-body { + padding-block-end: 0 !important; +} + +/* + * Keycloak's base keycloak.v2 theme applies a CSS filter (invert/sepia/ + * hue-rotate) to non-google provider icons to recolor their default black + * fill into Keycloak's own blue -- it repaints the rendered pixels after fill, + * so it fights the fixed icon color below and must be cancelled explicitly. + */ +#kc-social-providers svg:not(.google) { + filter: none !important; +} + +/* + * Google ships a multi-color logo with per-path fills; every other provider + * (github, etc.) draws a single path with no fill. The button text/border is + * blue (tertiary style), but the icon should read as neutral ink -- dark on + * the light card, light on the dark card -- not the accent blue. + */ +#kc-social-providers svg:not(.google) path { + fill: #151515 !important; +} + +/* Logout confirmation page -- "Do you want to log out?" text renders flush + against the Log out button with no separating space by default. */ +#kc-logout-confirm p.instruction { + margin: 0 0 1.5rem !important; +} + /* Alert styling */ .pf-v5-c-alert { background-color: #ffffff !important; @@ -382,6 +479,8 @@ h1.pf-v5-c-title.pf-m-3xl, } .pf-v5-c-button.pf-m-control::after { + border-top-color: #3c3f42 !important; + border-right-color: #3c3f42 !important; border-bottom-color: #3c3f42 !important; } @@ -475,4 +574,24 @@ h1.pf-v5-c-title.pf-m-3xl, .pf-v5-c-login__footer { color: #a2a2a2; } + + #kc-social-providers .pf-v5-c-button.pf-m-secondary { + --pf-v5-c-button--m-secondary--after--BorderColor: #73bcf7; + --pf-v5-c-button--m-secondary--hover--after--BorderColor: #bee1f4; + --pf-v5-c-button--m-secondary--focus--after--BorderColor: #bee1f4; + --pf-v5-c-button--m-secondary--active--after--BorderColor: #bee1f4; + background-color: transparent !important; + color: #73bcf7 !important; + } + + #kc-social-providers .pf-v5-c-button.pf-m-secondary:hover, + #kc-social-providers .pf-v5-c-button.pf-m-secondary:focus, + #kc-social-providers .pf-v5-c-button.pf-m-secondary:active { + background-color: transparent !important; + color: #bee1f4 !important; + } + + #kc-social-providers svg:not(.google) path { + fill: #e0e0e0 !important; + } } diff --git a/deploy/e2e/reaper/cronjob.yaml b/deploy/e2e/reaper/cronjob.yaml new file mode 100644 index 000000000..b9017e53b --- /dev/null +++ b/deploy/e2e/reaper/cronjob.yaml @@ -0,0 +1,70 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pr-env-reaper + namespace: hypershell-pr-reaper + labels: + app.kubernetes.io/part-of: hypershell + app.kubernetes.io/name: pr-env-reaper +spec: + # Hourly. Expiry is a per-namespace annotation, so the exact cadence only + # bounds how long an expired environment lingers, not correctness. Edit this + # single field to change how often the reaper runs. + schedule: "0 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 300 + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 600 + template: + metadata: + labels: + app.kubernetes.io/part-of: hypershell + app.kubernetes.io/name: pr-env-reaper + spec: + serviceAccountName: pr-env-reaper + restartPolicy: Never + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + # OpenShift SCC assigns a numeric UID. On a non-OpenShift cluster the + # ose-cli image must resolve to a non-root user or the pod fails + # admission (runAsNonRoot is set without runAsUser). + containers: + - name: reaper + # OVERRIDE: any image carrying bash + a Kubernetes CLI. ose-cli + # ships `oc`; PR_ENV_KUBECTL selects the binary the script calls. + image: registry.redhat.io/openshift4/ose-cli@sha256:f898cc139974e7753982d864b1351d28069e778bab9be89dcbf7b910e239fc01 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "/opt/reaper/reap-pr-environments.sh"] + env: + - name: PR_ENV_KUBECTL + value: oc + # Flip to "true" to log decisions without deleting anything. + - name: PR_ENV_REAP_DRY_RUN + value: "false" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + volumeMounts: + - name: reaper-scripts + mountPath: /opt/reaper + readOnly: true + volumes: + - name: reaper-scripts + configMap: + name: pr-env-reaper-scripts + defaultMode: 0555 diff --git a/deploy/e2e/reaper/kustomization.yaml b/deploy/e2e/reaper/kustomization.yaml new file mode 100644 index 000000000..7748249f8 --- /dev/null +++ b/deploy/e2e/reaper/kustomization.yaml @@ -0,0 +1,29 @@ +# Ephemeral pull-request environment reaper (ephemeral-pr-environments.spec.md, +# Timebox and Reaping). Apply out-of-band to the shared target cluster -- by hand +# or through Argo CD -- so expired pull-request environments are reclaimed +# independently of any CI run: +# +# kustomize build --load-restrictor=LoadRestrictionsNone deploy/e2e/reaper \ +# | oc apply -f - +# +# The --load-restrictor=LoadRestrictionsNone flag is required because the reaper +# ConfigMap is generated from the canonical scripts under scripts/ci/ (rather +# than a duplicated copy), which live above this directory. This mirrors how +# scripts/cluster renders the OpenShift overlay. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - cronjob.yaml + +configMapGenerator: + - name: pr-env-reaper-scripts + files: + - ../../../scripts/ci/pr-env-lib.sh + - ../../../scripts/ci/reap-pr-environments.sh + +# The reaper reads the pinned script content; a rolling hash would force a +# needless CronJob update on unrelated kustomize edits. +generatorOptions: + disableNameSuffixHash: true diff --git a/deploy/e2e/reaper/rbac.yaml b/deploy/e2e/reaper/rbac.yaml new file mode 100644 index 000000000..cbc027d70 --- /dev/null +++ b/deploy/e2e/reaper/rbac.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: hypershell-pr-reaper + labels: + app.kubernetes.io/part-of: hypershell + app.kubernetes.io/managed-by: hypershell-lifecycle +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pr-env-reaper + namespace: hypershell-pr-reaper + labels: + app.kubernetes.io/part-of: hypershell + app.kubernetes.io/name: pr-env-reaper +--- +# The reaper needs to list every namespace, read its ownership labels and +# expires-at annotation, and delete an expired pull-request environment's +# namespace group and that environment's cluster-scoped RBAC. Kubernetes cannot +# label-scope cluster-wide delete, so this ClusterRole can delete ANY namespace +# or ClusterRole/ClusterRoleBinding. Safety is the pr_env_is_reapable predicate +# in scripts/ci/pr-env-lib.sh (prefix hypershell-ci-pr-, owned=true, env id +# pr-, expires-at in the past, never reserved names). That predicate is +# unit-tested; a regression in it would be cluster-wide. Deploy this only on +# the dedicated PR-environments cluster, not onto a shared stage/prod hub. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hypershell-pr-env-reaper + labels: + app.kubernetes.io/part-of: hypershell + app.kubernetes.io/name: pr-env-reaper +rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "clusterrolebindings"] + verbs: ["get", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hypershell-pr-env-reaper + labels: + app.kubernetes.io/part-of: hypershell + app.kubernetes.io/name: pr-env-reaper +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hypershell-pr-env-reaper +subjects: + - kind: ServiceAccount + name: pr-env-reaper + namespace: hypershell-pr-reaper diff --git a/scripts/ci/pr-env-lib.sh b/scripts/ci/pr-env-lib.sh new file mode 100755 index 000000000..a83605ced --- /dev/null +++ b/scripts/ci/pr-env-lib.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# pr-env-lib.sh - pure helpers for the ephemeral pull-request environment +# workflow and its out-of-band reaper (ephemeral-pr-environments.spec.md, +# HYPERSHELL-240). +# +# This file holds ONLY pure, side-effect-free functions so both the CI workflow +# (scripts/ci/*.sh) and the in-cluster reaper (scripts/ci/reap-pr-environments.sh, +# shipped to the cluster via deploy/e2e/reaper) can share one definition of the +# per-PR namespace naming, the timebox, the ownership labels, and the reaper +# match predicate. It performs no cluster calls, so it is unit-tested without a +# cluster by scripts/ci/pr-env-lib_test.sh. Source it; do not execute it. + +# --- Ownership labels + timebox annotation (must match the OpenShift lifecycle +# driver in scripts/cluster/drivers/openshift.sh so status and cleanup tooling +# stay one selector set). --- +PR_ENV_NS_PREFIX="hypershell-ci-pr-" +PR_ENV_OWNED_LABEL="hypershell.redhat.io/owned" +PR_ENV_ENVIRONMENT_LABEL="hypershell.redhat.io/environment" +PR_ENV_MANAGED_LABEL="app.kubernetes.io/managed-by" +PR_ENV_MANAGED_VALUE="hypershell-lifecycle" +PR_ENV_PART_OF_LABEL="app.kubernetes.io/part-of" +PR_ENV_PART_OF_VALUE="hypershell" +PR_ENV_EXPIRES_ANNOTATION="hypershell.redhat.io/expires-at" +# Control-plane stamps on gateway and ManagedDatabase namespaces. Must match +# components/control-plane/internal/gateway/namespace.go. Distinct from +# PR_ENV_MANAGED_VALUE, which marks the platform/keycloak namespace group. +PR_ENV_CP_MANAGED_LABEL="hypershell.redhat.io/managed" +PR_ENV_CP_MANAGED_VALUE="true" +PR_ENV_CP_MANAGED_BY_LABEL="app.kubernetes.io/managed-by" +PR_ENV_CP_MANAGED_BY_VALUE="hypershell-control-plane" +PR_ENV_CP_INSTANCE_LABEL="hypershell.redhat.io/instance" + +# Default timebox in days. The workflow overrides this from a single documented +# setting (vars.PR_ENV_TIMEBOX_DAYS); the constant keeps the default in one place. +: "${PR_ENV_TIMEBOX_DAYS:=3}" + +# Stable hidden marker so later runs update the one access comment rather than +# post a new comment per run. +PR_ENV_COMMENT_MARKER='' + +# pr_env_namespace -> the platform namespace name. +pr_env_namespace() { + printf '%s%s' "${PR_ENV_NS_PREFIX}" "$1" +} + +# pr_env_keycloak_namespace -> the companion Keycloak +# namespace, matching keycloak_namespace_for in scripts/cluster/lib.sh. +pr_env_keycloak_namespace() { + printf '%s-keycloak' "$1" +} + +# pr_env_environment_id -> the environment identifier stamped into +# hypershell.redhat.io/environment. The pr- prefix distinguishes pull-request +# environments from local `make openshift-up` environments (opaque ids). +pr_env_environment_id() { + printf 'pr-%s' "$1" +} + +# pr_env_is_reserved_namespace - true for cluster-reserved namespaces the +# reaper must never delete. Mirrors is_reserved_cluster_namespace in +# scripts/cluster/lib.sh; duplicated (not sourced) so the reaper container needs +# only this one file. +pr_env_is_reserved_namespace() { + case "$1" in + default|openshift|kube-system|kube-public|kube-node-lease) return 0 ;; + kube-*|openshift-*) return 0 ;; + esac + return 1 +} + +# pr_env_epoch_to_rfc3339 -> RFC 3339 UTC timestamp, portable +# across GNU date (Linux/CI/reaper container) and BSD date (macOS). +pr_env_epoch_to_rfc3339() { + local epoch="$1" + if date -u -r "${epoch}" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null; then + return 0 + fi + date -u -d "@${epoch}" +%Y-%m-%dT%H:%M:%SZ +} + +# pr_env_rfc3339_to_epoch -> epoch seconds, or empty + non-zero when +# the timestamp cannot be parsed. Portable across GNU and BSD date. +pr_env_rfc3339_to_epoch() { + local ts="$1" + [[ -n "${ts}" ]] || return 1 + if date -u -d "${ts}" +%s 2>/dev/null; then + return 0 + fi + date -u -j -f '%Y-%m-%dT%H:%M:%SZ' "${ts}" +%s 2>/dev/null +} + +pr_env_now_epoch() { + date -u +%s +} + +# pr_env_expires_at [now-epoch] -> RFC 3339 UTC timestamp in the +# future. now-epoch is injectable for deterministic tests. +pr_env_expires_at() { + local days="$1" + local now="${2:-$(pr_env_now_epoch)}" + pr_env_epoch_to_rfc3339 $(( now + days * 86400 )) +} + +# pr_env_is_reapable [now-epoch] +# +# The single reaper match predicate (Timebox and Reaping requirement). Returns 0 +# (delete this namespace group) only when ALL hold: +# - name is prefixed hypershell-ci-pr- +# - name is not a reserved cluster namespace +# - hypershell.redhat.io/owned == true +# - hypershell.redhat.io/environment == pr- +# - hypershell.redhat.io/expires-at is present, parseable, and has passed +# Anything else (local openshift-up envs, unlabeled namespaces, an env id that is +# not pr-*, a still-in-the-future or missing expiry) is retained. Because every +# deploying run refreshes expires-at, an actively worked pull request never +# satisfies the expiry clause and is never reaped mid-flight. +pr_env_is_reapable() { + local name="$1" owned="$2" env_id="$3" expires_at="$4" + local now="${5:-$(pr_env_now_epoch)}" + [[ "${name}" == "${PR_ENV_NS_PREFIX}"* ]] || return 1 + if pr_env_is_reserved_namespace "${name}"; then + return 1 + fi + [[ "${owned}" == "true" ]] || return 1 + [[ "${env_id}" =~ ^pr-[0-9]+$ ]] || return 1 + local exp + exp="$(pr_env_rfc3339_to_epoch "${expires_at}")" || return 1 + [[ -n "${exp}" ]] || return 1 + (( now >= exp )) +} + +# pr_env_is_pr_platform_namespace - true for hypershell-ci-pr- +# only, not the companion -keycloak namespace. +pr_env_is_pr_platform_namespace() { + [[ "$1" =~ ^hypershell-ci-pr-[0-9]+$ ]] +} + +# pr_env_should_reap_instance_workload +# +# True when a control-plane-managed namespace is leftover from a pull-request +# platform project that no longer exists. platform-exists is the string "true" +# when kubectl can still get that instance's platform namespace. Local +# openshift-up instances (alice, hyp4, hyp5) and a still-live PR platform are +# retained. +pr_env_should_reap_instance_workload() { + local workload="$1" instance="$2" platform_exists="$3" + pr_env_is_pr_platform_namespace "${instance}" || return 1 + [[ "${platform_exists}" == "true" ]] && return 1 + [[ "${workload}" != "${instance}" ]] || return 1 + [[ "${workload}" != "${instance}-keycloak" ]] || return 1 + if pr_env_is_reserved_namespace "${workload}"; then + return 1 + fi + return 0 +} + +# pr_env_comment_access_facts +# +# Print the access-fact table and everything after it from an existing marked +# comment, or return non-zero when the body has no table. Namespaces, console +# URL, API Route URL, web-console Route URL, and the CLI login do not change +# from reconcile to reconcile, so a later deploying edit keeps this block +# instead of replacing the comment with the first-deploy placeholder. +pr_env_comment_access_facts() { + local body="${1:-}" + local prefix="${body%%"| Fact | Value |"*}" + [[ "${prefix}" != "${body}" ]] || return 1 + printf '%s' "| Fact | Value |${body#*"| Fact | Value |"}" +} + +# pr_env_comment_deploying_body [existing-body] +# +# Render the in-progress comment a deploy run posts immediately on start, +# before cluster login, deploy, or e2e. Carries the same hidden marker as +# pr_env_comment_body, so the later "ready" update edits this comment in +# place rather than posting a second one. +# +# First deploy (no existing-body, or an existing-body with no access-fact +# table): a placeholder with the head SHA and no access facts. That is +# normally the first comment this workflow ever adds, which keeps the access +# comment near the top of the pull request's timeline. +# +# Later reconcile (existing-body already has the access-fact table): the +# heading states the environment is updating to the new commit, and the +# existing table is retained so login details stay visible while the swap +# runs. +pr_env_comment_deploying_body() { + local head_sha="$1" + local existing="${2:-}" + local short_sha="${head_sha:0:7}" + local facts="" + if facts="$(pr_env_comment_access_facts "${existing}")"; then + cat < \ +# +# +# Render the pull-request access comment (Pull-Request Comment requirement). +# Carries the hidden marker so later runs find and update this comment, presents +# the same non-secret access facts `make openshift-up` prints, and contains no +# credential -- the `oc login` template uses `--web` against the OpenShift +# cluster API (not the HyperShell API Route) so OpenShift handles token +# retrieval interactively. is "true" for the per-commit update +# wording, "false" for the initial comment. +pr_env_comment_body() { + local pr_number="$1" head_sha="$2" platform_ns="$3" keycloak_ns="$4" + local console_url="$5" api_url="$6" web_url="$7" cluster_api_url="$8" updated="$9" + local short_sha="${head_sha:0:7}" + local heading + if [[ "${updated}" == "true" ]]; then + heading="HyperShell environment updated to commit \`${short_sha}\`" + else + heading="HyperShell environment ready" + fi + cat <CLI access + +\`\`\` +oc login --server=${cluster_api_url} --web +\`\`\` + + +EOF +} diff --git a/scripts/ci/pr-env-lib_test.sh b/scripts/ci/pr-env-lib_test.sh new file mode 100755 index 000000000..f0a2c13be --- /dev/null +++ b/scripts/ci/pr-env-lib_test.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# Unit tests for scripts/ci/pr-env-lib.sh. No cluster required. +# Run: bash scripts/ci/pr-env-lib_test.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=pr-env-lib.sh +source "${SCRIPT_DIR}/pr-env-lib.sh" + +PASS=0 +FAIL=0 + +assert_eq() { + local want="$1" got="$2" label="$3" + if [[ "${want}" == "${got}" ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + printf 'FAIL: %s (want=%q got=%q)\n' "${label}" "${want}" "${got}" + fi +} + +assert_reapable() { + local label="$1"; shift + if pr_env_is_reapable "$@"; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + printf 'FAIL: %s (expected reapable)\n' "${label}" + fi +} + +assert_not_reapable() { + local label="$1"; shift + if pr_env_is_reapable "$@"; then + FAIL=$((FAIL + 1)) + printf 'FAIL: %s (expected retained)\n' "${label}" + else + PASS=$((PASS + 1)) + fi +} + +# --- Namespace + identity derivation --- +assert_eq 'hypershell-ci-pr-232' "$(pr_env_namespace 232)" 'platform namespace from PR number' +assert_eq 'hypershell-ci-pr-232-keycloak' "$(pr_env_keycloak_namespace "$(pr_env_namespace 232)")" 'keycloak namespace derivation' +assert_eq 'pr-232' "$(pr_env_environment_id 232)" 'environment id from PR number' + +# The platform namespace must remain an RFC 1123 label within 54 chars so the +# derived -keycloak name stays under 63. Even a large PR number fits easily. +big_ns="$(pr_env_namespace 999999)" +assert_eq 'true' "$([[ ${#big_ns} -le 54 ]] && echo true || echo false)" 'platform namespace within 54 chars' + +# --- Timebox round-trip (injected clock for determinism) --- +base=1000000000 # 2001-09-09T01:46:40Z +assert_eq '2001-09-09T01:46:40Z' "$(pr_env_epoch_to_rfc3339 "${base}")" 'epoch -> rfc3339' +assert_eq "${base}" "$(pr_env_rfc3339_to_epoch "$(pr_env_epoch_to_rfc3339 "${base}")")" 'rfc3339 -> epoch round-trip' +# 3-day expiry is exactly 3*86400 seconds ahead. +assert_eq "$(pr_env_epoch_to_rfc3339 $((base + 3 * 86400)))" "$(pr_env_expires_at 3 "${base}")" 'expires_at is now + days' + +# --- Reaper predicate --- +now=2000000000 +past="$(pr_env_epoch_to_rfc3339 $((now - 60)))" +future="$(pr_env_epoch_to_rfc3339 $((now + 3600)))" + +assert_reapable 'expired owned pr env' \ + 'hypershell-ci-pr-232' 'true' 'pr-232' "${past}" "${now}" +assert_not_reapable 'not yet expired' \ + 'hypershell-ci-pr-232' 'true' 'pr-232' "${future}" "${now}" +assert_not_reapable 'missing expiry annotation' \ + 'hypershell-ci-pr-232' 'true' 'pr-232' '' "${now}" +assert_not_reapable 'not owned' \ + 'hypershell-ci-pr-232' 'false' 'pr-232' "${past}" "${now}" +# Local `make openshift-up` env: right owner labels but opaque (non pr-*) id. +assert_not_reapable 'local openshift-up env (uuid id)' \ + 'hypershell-ci-pr-232' 'true' '3f9a1c2e-uuid' "${past}" "${now}" +# A HyperShell env that is not a pr namespace at all. +assert_not_reapable 'non pr-prefixed namespace' \ + 'my-dev-namespace' 'true' 'pr-232' "${past}" "${now}" +# Env id must be pr-, not pr-anything. +assert_not_reapable 'env id pr- without a number' \ + 'hypershell-ci-pr-232' 'true' 'pr-branchname' "${past}" "${now}" +# Reserved names are refused even if they somehow carry the labels/prefix. +assert_not_reapable 'reserved openshift- namespace refused' \ + 'openshift-config' 'true' 'pr-1' "${past}" "${now}" + +assert_eq 'true' "$(pr_env_is_pr_platform_namespace 'hypershell-ci-pr-267' && echo true || echo false)" \ + 'pr platform namespace matches' +assert_eq 'false' "$(pr_env_is_pr_platform_namespace 'hypershell-ci-pr-267-keycloak' && echo true || echo false)" \ + 'keycloak companion is not a pr platform namespace' +assert_eq 'false' "$(pr_env_is_pr_platform_namespace 'hyp5' && echo true || echo false)" \ + 'hub namespace is not a pr platform namespace' + +if pr_env_should_reap_instance_workload 'openshell-aaa' 'hypershell-ci-pr-267' 'false'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: leftover pr-267 gateway should be reaped when platform is gone' +fi +if pr_env_should_reap_instance_workload 'openshell-aaa' 'hypershell-ci-pr-267' 'true'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: live pr-267 gateway should be retained' +else + PASS=$((PASS + 1)) +fi +if pr_env_should_reap_instance_workload 'openshell-aaa' 'hyp5' 'false'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: hyp5 gateway should be retained even if platform lookup fails' +else + PASS=$((PASS + 1)) +fi +if pr_env_should_reap_instance_workload 'openshell-aaa' 'alice' 'false'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: local openshift-up gateway should be retained' +else + PASS=$((PASS + 1)) +fi +if pr_env_should_reap_instance_workload 'hypershell-ci-pr-267' 'hypershell-ci-pr-267' 'false'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: instance leftover path must not delete the platform project itself' +else + PASS=$((PASS + 1)) +fi + +# --- Deploying placeholder comment (posted before the ready comment) --- +deploying_body="$(pr_env_comment_deploying_body abcdef1234567)" +case "${deploying_body}" in + *""*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: deploying comment missing hidden marker' ;; +esac +case "${deploying_body}" in + *'abcdef1'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: deploying comment missing short SHA' ;; +esac +case "${deploying_body}" in + *'| Fact | Value |'*) FAIL=$((FAIL + 1)); echo 'FAIL: first-deploy placeholder must not include the access-fact table' ;; + *) PASS=$((PASS + 1)) ;; +esac +case "${deploying_body}" in + *'Deploying commit `abcdef1` to an ephemeral OpenShift environment.'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: first-deploy placeholder missing deploying wording' ;; +esac +case "${deploying_body}" in + *'may not be fully responsive'*) FAIL=$((FAIL + 1)); echo 'FAIL: first-deploy placeholder must not warn about an existing environment' ;; + *) PASS=$((PASS + 1)) ;; +esac + +# A later reconcile must keep the existing table: those facts do not change +# from run to run, and wiping them hides login details for the whole swap. +ready_body="$(pr_env_comment_body 232 abcdef1234567 hypershell-ci-pr-232 hypershell-ci-pr-232-keycloak \ + https://console.example.com https://api.pr-232.example.com https://web.pr-232.example.com \ + https://api.cluster.example.com:6443 false)" +updating_body="$(pr_env_comment_deploying_body fffffff111111 "${ready_body}")" +case "${updating_body}" in + *""*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment missing hidden marker' ;; +esac +case "${updating_body}" in + *'## HyperShell environment updating to commit `fffffff`'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment missing updating heading' ;; +esac +case "${updating_body}" in + *'may not be fully responsive during the update'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment missing unresponsive-during-update note' ;; +esac +case "${updating_body}" in + *'| Fact | Value |'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment dropped the access-fact table' ;; +esac +case "${updating_body}" in + *'| Namespaces | Platform: `hypershell-ci-pr-232` Keycloak: `hypershell-ci-pr-232-keycloak` |'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment dropped namespace facts' ;; +esac +case "${updating_body}" in + *'| OpenShift console | https://console.example.com |'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment dropped console URL' ;; +esac +case "${updating_body}" in + *'oc login --server=https://api.cluster.example.com:6443 --web'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment dropped CLI login' ;; +esac +case "${updating_body}" in + *'abcdef1'*) FAIL=$((FAIL + 1)); echo 'FAIL: updating comment kept the previous commit SHA' ;; + *) PASS=$((PASS + 1)) ;; +esac +# A still-in-progress first deploy has no table yet; a cancelled run that +# never reached ready must keep posting the no-facts placeholder. +still_deploying="$(pr_env_comment_deploying_body fffffff111111 "${deploying_body}")" +case "${still_deploying}" in + *'| Fact | Value |'*) FAIL=$((FAIL + 1)); echo 'FAIL: in-progress first deploy must not invent a table' ;; + *) PASS=$((PASS + 1)) ;; +esac +case "${still_deploying}" in + *'Deploying commit `fffffff` to an ephemeral OpenShift environment.'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: in-progress first deploy missing updated SHA' ;; +esac +# A mid-reconcile comment already has the table; the next deploying edit +# must keep it and only advance the SHA. +second_update="$(pr_env_comment_deploying_body 1234567890abc "${updating_body}")" +case "${second_update}" in + *'## HyperShell environment updating to commit `1234567`'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: second updating comment missing new SHA heading' ;; +esac +case "${second_update}" in + *'| Namespaces | Platform: `hypershell-ci-pr-232` Keycloak: `hypershell-ci-pr-232-keycloak` |'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: second updating comment dropped namespace facts' ;; +esac +case "${second_update}" in + *'fffffff'*) FAIL=$((FAIL + 1)); echo 'FAIL: second updating comment kept the previous commit SHA' ;; + *) PASS=$((PASS + 1)) ;; +esac + +# --- Comment body --- +body="$(pr_env_comment_body 232 abcdef1234567 hypershell-ci-pr-232 hypershell-ci-pr-232-keycloak \ + https://console.example.com https://api.pr-232.example.com https://web.pr-232.example.com \ + https://api.cluster.example.com:6443 false)" +case "${body}" in + *""*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: comment body missing hidden marker' ;; +esac +case "${body}" in + *'abcdef1'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: comment body missing short SHA' ;; +esac +# The CLI template must use --web against the cluster API, never the app Route. +case "${body}" in + *'oc login --server=https://api.cluster.example.com:6443 --web'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: comment body oc login missing cluster API --web' ;; +esac +case "${body}" in + *'oc login --server=https://api.pr-232.example.com'*) FAIL=$((FAIL + 1)); echo 'FAIL: oc login used app API Route' ;; + *) PASS=$((PASS + 1)) ;; +esac +updated_body="$(pr_env_comment_body 232 abcdef1234567 ns ns-keycloak c a w https://api.cluster.example.com true)" +case "${updated_body}" in + *'updated to commit'*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); echo 'FAIL: updated comment missing update wording' ;; +esac + +printf 'pr-env-lib tests: %d passed, %d failed\n' "$PASS" "$FAIL" +[[ "$FAIL" -eq 0 ]] diff --git a/scripts/ci/read-e2e-client-secret.sh b/scripts/ci/read-e2e-client-secret.sh new file mode 100755 index 000000000..83a9214d1 --- /dev/null +++ b/scripts/ci/read-e2e-client-secret.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# read-e2e-client-secret.sh - read the hypershell-e2e confidential client secret +# from the deployed per-PR Keycloak namespace (ephemeral-pr-environments.spec.md: +# Automated E2E Authentication). +# +# The e2e suite authenticates through the hypershell-e2e client (client +# credentials for the admin path, token exchange for the developer path), never +# through a brokered GitHub user's password grant. The client secret is per-PR: +# it is the same value the workflow put in the hypershell-github-oauth Secret in +# the Keycloak namespace, which the render-realm-config init container splices +# into the hypershell-e2e client via the ${HYPERSHELL_E2E_CLIENT_SECRET} +# placeholder. This script prints that secret to stdout so the workflow can +# mask it and export +# E2E_OIDC_SA_CLIENT_SECRET; it must never be echoed into logs, the pull-request +# comment, or a public artifact. +# +# Environment: +# PR_NUMBER pull-request number (required) +# PR_ENV_E2E_SECRET_NAME Secret name (default: hypershell-github-oauth) +# PR_ENV_E2E_SECRET_KEY Secret data key (default: e2e-client-secret) +# PR_ENV_KUBECTL kubectl/oc binary (default: oc) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=pr-env-lib.sh +source "${SCRIPT_DIR}/pr-env-lib.sh" + +KUBECTL="${PR_ENV_KUBECTL:-oc}" +: "${PR_NUMBER:?PR_NUMBER is required}" +secret_name="${PR_ENV_E2E_SECRET_NAME:-hypershell-github-oauth}" +secret_key="${PR_ENV_E2E_SECRET_KEY:-e2e-client-secret}" + +platform_ns="$(pr_env_namespace "${PR_NUMBER}")" +keycloak_ns="$(pr_env_keycloak_namespace "${platform_ns}")" + +encoded="$("${KUBECTL}" get secret "${secret_name}" -n "${keycloak_ns}" \ + -o "jsonpath={.data.${secret_key}}" 2>/dev/null || true)" + +if [[ -z "${encoded}" ]]; then + { + echo "ERROR: could not read ${secret_key} from Secret ${secret_name} in ${keycloak_ns}." + echo "The workflow's 'Provision GitHub OAuth secret' step must create this Secret" + echo "(with the e2e-client-secret key) before Keycloak boots and e2e runs." + } >&2 + exit 1 +fi + +printf '%s' "${encoded}" | base64 -d diff --git a/scripts/ci/reap-pr-environments.sh b/scripts/ci/reap-pr-environments.sh new file mode 100755 index 000000000..8877f2df7 --- /dev/null +++ b/scripts/ci/reap-pr-environments.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# reap-pr-environments.sh - out-of-band reaper for ephemeral pull-request +# environments (ephemeral-pr-environments.spec.md, Timebox and Reaping). +# +# Runs independently of any CI run (as a CronJob on the target cluster, see +# deploy/e2e/reaper), so an abandoned pull request's environment is reclaimed +# even when no further CI runs for that pull request. It deletes a namespace when +# and only when pr_env_is_reapable is true: prefixed hypershell-ci-pr-, owned by +# HyperShell, environment id pr-, and past its +# hypershell.redhat.io/expires-at. Because every deploying run refreshes that +# annotation, an actively worked pull request is never reaped mid-flight; a +# namespace with no activity for the timebox falls past its expiry and is removed. +# +# The reaper matches the platform and the -keycloak namespaces independently +# (both carry the same labels and prefix), so one pass removes the whole group. +# Gateway and ManagedDatabase namespaces are siblings labeled +# hypershell.redhat.io/instance=; they are reaped with the platform +# project and again if that project is already gone. +# +# Environment: +# PR_ENV_KUBECTL kubectl/oc binary (default: kubectl) +# PR_ENV_REAP_DRY_RUN when "true", log the decisions but delete nothing +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=pr-env-lib.sh +source "${SCRIPT_DIR}/pr-env-lib.sh" + +KUBECTL="${PR_ENV_KUBECTL:-kubectl}" +DRY_RUN="${PR_ENV_REAP_DRY_RUN:-false}" + +log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } + +# List candidate namespaces (owned by HyperShell) as tab-separated +# nameownedenvironmentexpires-at. Narrowed by the owned label; +# the full predicate still runs per row. `if` guards protect against namespaces +# with no labels or no annotations (index on a nil map errors in go-template). +list_owned_namespaces() { + "${KUBECTL}" get namespaces \ + -l "${PR_ENV_OWNED_LABEL}=true" \ + -o go-template='{{range .items}}{{.metadata.name}}{{"\t"}}{{if .metadata.labels}}{{index .metadata.labels "hypershell.redhat.io/owned"}}{{end}}{{"\t"}}{{if .metadata.labels}}{{index .metadata.labels "hypershell.redhat.io/environment"}}{{end}}{{"\t"}}{{if .metadata.annotations}}{{index .metadata.annotations "hypershell.redhat.io/expires-at"}}{{end}}{{"\n"}}{{end}}' +} + +# nameinstance for namespaces the control plane stamped. +list_control_plane_managed_namespaces() { + "${KUBECTL}" get namespaces \ + -l "${PR_ENV_CP_MANAGED_LABEL}=${PR_ENV_CP_MANAGED_VALUE},${PR_ENV_CP_MANAGED_BY_LABEL}=${PR_ENV_CP_MANAGED_BY_VALUE}" \ + -o go-template='{{range .items}}{{.metadata.name}}{{"\t"}}{{if .metadata.labels}}{{index .metadata.labels "hypershell.redhat.io/instance"}}{{end}}{{"\n"}}{{end}}' +} + +platform_namespace_exists() { + "${KUBECTL}" get namespace "$1" >/dev/null 2>&1 +} + +# Delete this environment's cluster-scoped RBAC, mirroring cluster_down in +# scripts/cluster/drivers/openshift.sh. Only the platform namespace owns the +# ${ns}-dev-* cluster RBAC; the -keycloak namespace has none. +delete_cluster_rbac() { + local ns="$1" + local prefix="${ns}-dev-" + local kind name + for kind in clusterrolebinding clusterrole; do + for name in "${prefix}hypershell-controller-scc-bind" "${prefix}hypershell-controller"; do + if [[ "${DRY_RUN}" == "true" ]]; then + log " DRY-RUN would delete ${kind}/${name}" + else + "${KUBECTL}" delete "${kind}" "${name}" --ignore-not-found >/dev/null 2>&1 || true + fi + done + done +} + +reap_namespace() { + local ns="$1" + if [[ "${DRY_RUN}" == "true" ]]; then + log " DRY-RUN would delete namespace ${ns}" + return 0 + fi + # --wait=false: do not block the reaper on finalizers; the delete is recorded + # and the namespace terminates asynchronously. + if "${KUBECTL}" delete namespace "${ns}" --ignore-not-found --wait=false >/dev/null 2>&1; then + log " deleted namespace ${ns}" + else + log " WARNING failed to delete namespace ${ns}" + return 1 + fi +} + +# Delete sibling gateway/database namespaces this platform instance stamped. +reap_instance_workloads() { + local instance="$1" + local rows name inst + if [[ -z "${instance}" ]]; then + log " WARNING refusing to reap instance workloads with an empty instance identity" + return 1 + fi + if ! rows="$(list_control_plane_managed_namespaces)"; then + log " WARNING could not list instance-managed namespaces for ${instance}" + return 1 + fi + while IFS=$'\t' read -r name inst; do + [[ -n "${name}" ]] || continue + [[ "${inst}" == "${instance}" ]] || continue + if [[ "${name}" == "${instance}" || "${name}" == "${instance}-keycloak" ]]; then + continue + fi + log " instance workload ${name} (instance=${instance})" + reap_namespace "${name}" || true + done <<< "${rows}" +} + +reap_leftover_instance_workloads() { + local rows name inst exists + if ! rows="$(list_control_plane_managed_namespaces)"; then + log "ERROR: could not list control-plane managed namespaces via ${KUBECTL}" + return 1 + fi + while IFS=$'\t' read -r name inst; do + [[ -n "${name}" ]] || continue + exists="false" + if [[ -n "${inst}" ]] && platform_namespace_exists "${inst}"; then + exists="true" + fi + if pr_env_should_reap_instance_workload "${name}" "${inst}" "${exists}"; then + log "REAP leftover ${name} (instance=${inst}, platform absent)" + if reap_namespace "${name}"; then + reaped=$((reaped + 1)) + else + failed=$((failed + 1)) + fi + fi + done <<< "${rows}" +} + +main() { + local now considered=0 reaped=0 retained=0 failed=0 + now="$(pr_env_now_epoch)" + log "pr-env reaper: scanning owned namespaces (dry_run=${DRY_RUN})" + + local rows + if ! rows="$(list_owned_namespaces)"; then + log "ERROR: could not list namespaces via ${KUBECTL}" + return 1 + fi + + local name owned env_id expires + while IFS=$'\t' read -r name owned env_id expires; do + [[ -n "${name}" ]] || continue + considered=$((considered + 1)) + if pr_env_is_reapable "${name}" "${owned}" "${env_id}" "${expires}" "${now}"; then + log "REAP ${name} (env=${env_id}, expired at ${expires})" + if reap_namespace "${name}"; then + # Only the platform namespace owns cluster RBAC and instance-stamped + # gateway/database namespaces; skip the -keycloak half. + if [[ "${name}" != *-keycloak ]]; then + delete_cluster_rbac "${name}" + reap_instance_workloads "${name}" + fi + reaped=$((reaped + 1)) + else + failed=$((failed + 1)) + fi + else + retained=$((retained + 1)) + fi + done <<< "${rows}" + + reap_leftover_instance_workloads + + log "pr-env reaper: considered=${considered} reaped=${reaped} retained=${retained} failed=${failed}" + [[ "${failed}" -eq 0 ]] +} + +main "$@" diff --git a/scripts/ci/reap-pr-environments_test.sh b/scripts/ci/reap-pr-environments_test.sh new file mode 100755 index 000000000..3081ca4d1 --- /dev/null +++ b/scripts/ci/reap-pr-environments_test.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Unit test for scripts/ci/reap-pr-environments.sh with a stubbed kubectl. +# No cluster required. Run: bash scripts/ci/reap-pr-environments_test.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=pr-env-lib.sh +source "${SCRIPT_DIR}/pr-env-lib.sh" + +PASS=0 +FAIL=0 + +now="$(pr_env_now_epoch)" +PAST="$(pr_env_epoch_to_rfc3339 $((now - 3600)))" +FUTURE="$(pr_env_epoch_to_rfc3339 $((now + 3600)))" + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT +DELETED="${workdir}/deleted.txt" +: > "${DELETED}" + +# Canned namespace table the stub returns for `get namespaces` with the owned +# label. Columns: nameownedenvironmentexpires-at. Covers: expired +# PR pair (reap both halves), active PR (future expiry, retain), local +# openshift-up env (uuid id, retain), and an unlabeled-ish foreign env. +cat > "${workdir}/rows.tsv" < "${workdir}/cp_managed.tsv" < "${workdir}/live.txt" < "${workdir}/kubectl" <> "${DELETED}" + ;; + *) + # delete clusterrolebinding/clusterrole and anything else: succeed quietly. + exit 0 + ;; +esac +EOF +chmod +x "${workdir}/kubectl" + +PR_ENV_KUBECTL="${workdir}/kubectl" bash "${SCRIPT_DIR}/reap-pr-environments.sh" >/dev/null + +deleted_sorted="$(sort -u "${DELETED}" | tr '\n' ' ')" +expected='hypershell-ci-pr-232 hypershell-ci-pr-232-keycloak openshell-aaa openshell-ccc openshell-db-bbb ' +if [[ "${deleted_sorted}" == "${expected}" ]]; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + printf 'FAIL: reaper deleted the wrong set (got=%q want=%q)\n' "${deleted_sorted}" "${expected}" +fi + +# Dry-run must delete nothing. +: > "${DELETED}" +PR_ENV_KUBECTL="${workdir}/kubectl" PR_ENV_REAP_DRY_RUN=true bash "${SCRIPT_DIR}/reap-pr-environments.sh" >/dev/null +if [[ -s "${DELETED}" ]]; then + FAIL=$((FAIL + 1)) + echo 'FAIL: dry-run deleted namespaces' +else + PASS=$((PASS + 1)) +fi + +printf 'reaper tests: %d passed, %d failed\n' "$PASS" "$FAIL" +[[ "$FAIL" -eq 0 ]] diff --git a/scripts/ci/stamp-pr-env.sh b/scripts/ci/stamp-pr-env.sh new file mode 100755 index 000000000..dd28ff16f --- /dev/null +++ b/scripts/ci/stamp-pr-env.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# stamp-pr-env.sh - stamp the per-PR namespace group with the ownership labels +# and the timebox annotation after a successful `make openshift-up` +# (ephemeral-pr-environments.spec.md: Per-Pull-Request Environment Identity, +# Timebox and Reaping). +# +# `make openshift-up` assigns an opaque environment id and does NOT write the +# timebox; this script overwrites the environment id with pr- so the +# reaper can attribute the group to its pull request, and stamps +# hypershell.redhat.io/expires-at in the future. It fails closed: if +# labeling or annotating either namespace fails, the whole workflow must fail and +# NOT leave an unlabeled or un-timeboxed environment (the local-dev +# warn-and-continue path does not apply to CI). +# +# Environment: +# PR_NUMBER pull-request number (required) +# PR_ENV_TIMEBOX_DAYS timebox in days (default 3) +# PR_ENV_KUBECTL kubectl/oc binary (default: oc) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=pr-env-lib.sh +source "${SCRIPT_DIR}/pr-env-lib.sh" + +KUBECTL="${PR_ENV_KUBECTL:-oc}" +: "${PR_NUMBER:?PR_NUMBER is required}" + +platform_ns="$(pr_env_namespace "${PR_NUMBER}")" +keycloak_ns="$(pr_env_keycloak_namespace "${platform_ns}")" +env_id="$(pr_env_environment_id "${PR_NUMBER}")" +expires_at="$(pr_env_expires_at "${PR_ENV_TIMEBOX_DAYS:-3}")" + +stamp_namespace() { + local ns="$1" + echo "Stamping ${ns} (environment=${env_id}, expires-at=${expires_at})" + "${KUBECTL}" label namespace "${ns}" \ + "${PR_ENV_OWNED_LABEL}=true" \ + "${PR_ENV_ENVIRONMENT_LABEL}=${env_id}" \ + "${PR_ENV_MANAGED_LABEL}=${PR_ENV_MANAGED_VALUE}" \ + "${PR_ENV_PART_OF_LABEL}=${PR_ENV_PART_OF_VALUE}" \ + --overwrite + "${KUBECTL}" annotate namespace "${ns}" \ + "${PR_ENV_EXPIRES_ANNOTATION}=${expires_at}" \ + --overwrite +} + +stamp_namespace "${platform_ns}" +stamp_namespace "${keycloak_ns}" + +# Publish the resolved facts for later workflow steps (comment, summary). +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "platform_namespace=${platform_ns}" + echo "keycloak_namespace=${keycloak_ns}" + echo "environment_id=${env_id}" + echo "expires_at=${expires_at}" + } >> "${GITHUB_OUTPUT}" +fi + +echo "Stamped namespace group ${platform_ns} + ${keycloak_ns}" diff --git a/scripts/ci/swap-openshift-images-by-digest.sh b/scripts/ci/swap-openshift-images-by-digest.sh new file mode 100755 index 000000000..69b9568d1 --- /dev/null +++ b/scripts/ci/swap-openshift-images-by-digest.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# swap-openshift-images-by-digest.sh - inject the pull request's component images +# into the deployed environment by immutable digest +# (ephemeral-pr-environments.spec.md: Image Gating and Swap). +# +# The workflow gates on the Konflux builds for the head commit and passes each +# built image reference here. This script resolves each reference to its manifest +# digest and rolls the deployment to repo@sha256:, so the environment +# runs exactly the artifact CI verified and a later re-push of a mutable tag +# cannot change it. +# +# Built pull-request images (tag on-pr-) MUST resolve to a digest; a missing +# skopeo/oc inspect or a registry that hides the digest is a hard error, not a +# silent tag swap. Baseline images (unchanged components) may fall back to the +# tag when no digest is available; that fallback is recorded in the run output +# rather than silent, matching the spec's last-resort clause. +# +# Environment: +# OPENSHIFT_NAMESPACE target platform namespace (required) +# API_SERVER_IMAGE api-server image ref (optional) +# CONTROL_PLANE_IMAGE control-plane image ref (optional) +# WEB_CONSOLE_IMAGE web-console image ref (optional) +# ROLLOUT_TIMEOUT per-deployment rollout ceiling (default 300s) +# PR_ENV_KUBECTL kubectl/oc binary (default: oc) +set -euo pipefail + +KUBECTL="${PR_ENV_KUBECTL:-oc}" +: "${OPENSHIFT_NAMESPACE:?OPENSHIFT_NAMESPACE is required}" +: "${ROLLOUT_TIMEOUT:=300s}" + +# inspect_digest -> sha256: or empty when neither skopeo +# nor oc can resolve one. +inspect_digest() { + local ref="$1" + local digest="" + if command -v skopeo >/dev/null 2>&1; then + digest="$(skopeo inspect --format '{{.Digest}}' "docker://${ref}" 2>/dev/null || true)" + fi + if [[ "${digest}" != sha256:* ]] && command -v "${KUBECTL}" >/dev/null 2>&1; then + # `oc image info -o jsonpath` is not supported on several oc versions and + # silently returns empty. Parse the JSON digest instead. + digest="$("${KUBECTL}" image info "${ref}" -o json 2>/dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("digest") or "")' 2>/dev/null || true)" + fi + printf '%s' "${digest}" +} + +# resolve_by_digest -> repo@sha256:. Built on-pr- tags +# fail closed when no digest is available. Other refs (baseline) fall back to +# the original tag with a recorded warning. An already-pinned @sha256 ref is +# returned unchanged. +resolve_by_digest() { + local ref="$1" + if [[ "${ref}" == *@sha256:* ]]; then + printf '%s' "${ref}" + return 0 + fi + local repo="${ref%:*}" + local digest + digest="$(inspect_digest "${ref}")" + if [[ "${digest}" == sha256:* ]]; then + printf '%s@%s' "${repo}" "${digest}" + return 0 + fi + if [[ "${ref}" == *":on-pr-"* ]]; then + echo "ERROR: no digest for built image ${ref}; refusing mutable-tag fallback." >&2 + echo "Install skopeo (or use oc image info) and ensure the runner can inspect the registry." >&2 + return 1 + fi + echo "WARNING: no digest available for ${ref}; falling back to the mutable tag" >&2 + printf '%s' "${ref}" +} + +swap_deployment() { + local deployment="$1" ref="$2" + shift 2 + local containers=("$@") + [[ -n "${ref}" ]] || return 0 + + local image + image="$(resolve_by_digest "${ref}")" + echo " ${deployment} -> ${image}" + + local set_args=() + local c + for c in "${containers[@]}"; do + set_args+=("${c}=${image}") + done + "${KUBECTL}" set image "deployment/${deployment}" "${set_args[@]}" -n "${OPENSHIFT_NAMESPACE}" + "${KUBECTL}" rollout status "deployment/${deployment}" -n "${OPENSHIFT_NAMESPACE}" --timeout="${ROLLOUT_TIMEOUT}" +} + +swap_pr_images() { + if [[ -z "${API_SERVER_IMAGE:-}" && -z "${CONTROL_PLANE_IMAGE:-}" && -z "${WEB_CONSOLE_IMAGE:-}" ]]; then + echo "No component image overrides set; environment keeps baseline images." + return 0 + fi + + echo "Swapping pull-request component images by digest into ${OPENSHIFT_NAMESPACE}" + swap_deployment hypershell-api-server "${API_SERVER_IMAGE:-}" api-server migrate + swap_deployment hypershell-controller "${CONTROL_PLANE_IMAGE:-}" controller + swap_deployment hypershell-web-console "${WEB_CONSOLE_IMAGE:-}" web-console + echo "Component image swap complete." +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + swap_pr_images +fi diff --git a/scripts/ci/swap-openshift-images-by-digest_test.sh b/scripts/ci/swap-openshift-images-by-digest_test.sh new file mode 100755 index 000000000..8b0bba84d --- /dev/null +++ b/scripts/ci/swap-openshift-images-by-digest_test.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Unit tests for scripts/ci/swap-openshift-images-by-digest.sh. No cluster +# required; skopeo and oc are stubbed on PATH. +# Run: bash scripts/ci/swap-openshift-images-by-digest_test.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OPENSHIFT_NAMESPACE=hypershell-ci-pr-test +# shellcheck source=swap-openshift-images-by-digest.sh +source "${SCRIPT_DIR}/swap-openshift-images-by-digest.sh" + +PASS=0 +FAIL=0 + +assert_eq() { + local want="$1" got="$2" label="$3" + if [[ "${want}" == "${got}" ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + printf 'FAIL: %s (want=%q got=%q)\n' "${label}" "${want}" "${got}" + fi +} + +assert_fails() { + local label="$1" + shift + if "$@" >/dev/null 2>&1; then + FAIL=$((FAIL + 1)) + printf 'FAIL: %s (expected non-zero)\n' "${label}" + else + PASS=$((PASS + 1)) + fi +} + +STUB_BIN="$(mktemp -d)" +cleanup() { rm -rf "${STUB_BIN}"; } +trap cleanup EXIT +PATH="${STUB_BIN}:${PATH}" + +# --- already pinned --- +assert_eq 'quay.io/org/img@sha256:abc' \ + "$(resolve_by_digest 'quay.io/org/img@sha256:abc')" \ + 'already-pinned digest is unchanged' + +# --- skopeo resolves --- +cat > "${STUB_BIN}/skopeo" <<'EOF' +#!/usr/bin/env bash +echo 'sha256:deadbeef' +EOF +chmod +x "${STUB_BIN}/skopeo" +assert_eq 'quay.io/org/img@sha256:deadbeef' \ + "$(resolve_by_digest 'quay.io/org/img:on-pr-abc123')" \ + 'on-pr tag pins via skopeo digest' + +# --- built image with no digest fails closed --- +cat > "${STUB_BIN}/skopeo" <<'EOF' +#!/usr/bin/env bash +exit 1 +EOF +chmod +x "${STUB_BIN}/skopeo" +# oc image info also missing / failing +cat > "${STUB_BIN}/oc" <<'EOF' +#!/usr/bin/env bash +exit 1 +EOF +chmod +x "${STUB_BIN}/oc" +KUBECTL=oc +assert_fails 'on-pr tag without digest is a hard error' \ + resolve_by_digest 'quay.io/org/img:on-pr-abc123' + +# --- baseline may fall back --- +got="$(resolve_by_digest 'quay.io/org/img:latest' 2>/dev/null || true)" +assert_eq 'quay.io/org/img:latest' "${got}" 'baseline tag may fall back when no digest' + +# --- oc image info used when skopeo is absent --- +rm -f "${STUB_BIN}/skopeo" +cat > "${STUB_BIN}/oc" <<'EOF' +#!/usr/bin/env bash +if [[ "$1" == "image" && "$2" == "info" ]]; then + echo '{"digest":"sha256:fromoc"}' + exit 0 +fi +exit 1 +EOF +chmod +x "${STUB_BIN}/oc" +assert_eq 'quay.io/org/img@sha256:fromoc' \ + "$(resolve_by_digest 'quay.io/org/img:on-pr-abc123')" \ + 'oc image info -o json pins when skopeo is missing' + +echo "swap-by-digest tests: ${PASS} passed, ${FAIL} failed" +[[ "${FAIL}" -eq 0 ]] diff --git a/scripts/ci/upsert-pr-comment.sh b/scripts/ci/upsert-pr-comment.sh new file mode 100755 index 000000000..d2b9c361e --- /dev/null +++ b/scripts/ci/upsert-pr-comment.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# upsert-pr-comment.sh - post or update the single pull-request access comment +# (ephemeral-pr-environments.spec.md: Pull-Request Comment and Access Handoff). +# +# Keeps exactly one comment current for the pull request by locating the comment +# carrying the hidden marker and editing it in place, rather than posting a new +# comment per run. Called twice per deploy run: once at the very start with +# PR_ENV_PHASE=deploying, and again once the environment is ready with the real +# access facts and an `oc login --web` template. The deploying phase posts a +# no-facts placeholder on first deploy (so the comment is normally first on the +# pull request and stays near the top of the timeline). On a later reconcile it +# updates the heading to the new commit and keeps the existing access-fact +# table, because those URLs and namespaces do not change from run to run. +# OpenShift handles token retrieval and refresh interactively, so no credential +# ever appears in the comment. +# +# Requires `gh` (authenticated via GH_TOKEN) and `jq`. +# +# Environment: +# GH_REPO / GITHUB_REPOSITORY owner/repo (gh reads GH_REPO) +# PR_NUMBER pull-request number (required) +# PR_HEAD_SHA head commit SHA (required) +# PR_ENV_PHASE "deploying" (in-progress heading; posted +# first, keeps an existing access-fact table) +# or "ready" (default; full access facts) +# PR_ENV_UPDATED "true" for the per-commit update wording +# ("ready" phase only) +# PLATFORM_NS / KEYCLOAK_NS namespace group ("ready" phase only) +# CONSOLE_URL / API_URL / WEB_URL / CLUSTER_API_URL access URLs +# ("ready" phase only) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=pr-env-lib.sh +source "${SCRIPT_DIR}/pr-env-lib.sh" + +: "${PR_NUMBER:?PR_NUMBER is required}" +: "${PR_HEAD_SHA:?PR_HEAD_SHA is required}" +repo="${GH_REPO:-${GITHUB_REPOSITORY:?GH_REPO or GITHUB_REPOSITORY is required}}" + +# Find an existing marked comment first (paginate; the marker is unique to +# this bot) so the deploying phase can keep its access-fact table. +existing_id="$(gh api --paginate \ + "repos/${repo}/issues/${PR_NUMBER}/comments" \ + --jq ".[] | select(.body | contains(\"${PR_ENV_COMMENT_MARKER}\")) | .id" \ + 2>/dev/null | head -n1 || true)" + +phase="${PR_ENV_PHASE:-ready}" +case "${phase}" in + deploying) + existing_body="" + if [[ -n "${existing_id}" ]]; then + existing_body="$(gh api "repos/${repo}/issues/comments/${existing_id}" --jq .body)" + fi + body="$(pr_env_comment_deploying_body "${PR_HEAD_SHA}" "${existing_body}")" + ;; + ready) + body="$(pr_env_comment_body \ + "${PR_NUMBER}" \ + "${PR_HEAD_SHA}" \ + "${PLATFORM_NS:-}" \ + "${KEYCLOAK_NS:-}" \ + "${CONSOLE_URL:-}" \ + "${API_URL:-}" \ + "${WEB_URL:-}" \ + "${CLUSTER_API_URL:-}" \ + "${PR_ENV_UPDATED:-false}")" + ;; + *) + echo "::error::Unknown PR_ENV_PHASE '${phase}' (want deploying or ready)" >&2 + exit 1 + ;; +esac + +if [[ -n "${existing_id}" ]]; then + echo "Updating existing access comment ${existing_id}" + gh api --method PATCH "repos/${repo}/issues/comments/${existing_id}" \ + -f body="${body}" >/dev/null +else + echo "Posting initial access comment" + gh api --method POST "repos/${repo}/issues/${PR_NUMBER}/comments" \ + -f body="${body}" >/dev/null +fi diff --git a/scripts/cluster/drivers/openshift.sh b/scripts/cluster/drivers/openshift.sh index 7c691a679..0bbd6a512 100755 --- a/scripts/cluster/drivers/openshift.sh +++ b/scripts/cluster/drivers/openshift.sh @@ -9,11 +9,36 @@ MANAGED_LABEL="app.kubernetes.io/managed-by" MANAGED_VALUE="hypershell-lifecycle" PART_OF_LABEL="app.kubernetes.io/part-of" PART_OF_VALUE="hypershell" +# Control-plane stamps on gateway and ManagedDatabase namespaces. Must match +# components/control-plane/internal/gateway/namespace.go (ManagedLabel, +# ManagedByValue, InstanceLabel). Distinct from MANAGED_VALUE above, which marks +# the platform/keycloak namespace group. +CP_MANAGED_LABEL="hypershell.redhat.io/managed" +CP_MANAGED_VALUE="true" +CP_MANAGED_BY_VALUE="hypershell-control-plane" +CP_INSTANCE_LABEL="hypershell.redhat.io/instance" oc_cli() { oc "$@" } +# Whether this environment brokers interactive login to GitHub instead of the +# realm's seeded admin/developer passwords +# (ephemeral-pr-environments.spec.md: GitHub-Brokered Keycloak Authentication). +# Brokered environments have no password grant, so seeding and the banner +# must use the hypershell-e2e service account instead of admin/admin. +github_idp_enabled() { + local enabled + enabled="$(oc_cli get secret hypershell-github-oauth -n "${OPENSHIFT_KEYCLOAK_NAMESPACE}" \ + -o jsonpath='{.data.idp-enabled}' 2>/dev/null | base64 -d 2>/dev/null || true)" + [[ "${enabled}" == "true" ]] +} + +hypershell_e2e_client_secret() { + oc_cli get secret hypershell-github-oauth -n "${OPENSHIFT_KEYCLOAK_NAMESPACE}" \ + -o jsonpath='{.data.e2e-client-secret}' 2>/dev/null | base64 -d 2>/dev/null || true +} + require_openshift_cluster() { if ! command -v oc >/dev/null 2>&1; then error "oc is not installed. Install the OpenShift CLI and retry." @@ -876,7 +901,10 @@ wait_for_named_rollout() { } wait_for_keycloak() { - wait_for_named_rollout keycloak "${OPENSHIFT_KEYCLOAK_NAMESPACE}" + # The shared e2e cluster can need to scale up a node for this pod (cluster + # autoscaler), which alone can take several minutes before it is even + # Scheduled. The default rollout timeout is too tight for that. + wait_for_named_rollout keycloak "${OPENSHIFT_KEYCLOAK_NAMESPACE}" 600s } configure_oidc_from_routes() { @@ -902,9 +930,16 @@ configure_oidc_from_routes() { OPENSHIFT_KC_HOSTNAME="https://${kc_host}" OPENSHIFT_OIDC_ISSUER="https://${kc_host}/realms/hypershell" - info "Setting Keycloak KC_HOSTNAME=${OPENSHIFT_KC_HOSTNAME}" - oc_cli set env deployment/keycloak -n "${OPENSHIFT_KEYCLOAK_NAMESPACE}" \ - "KC_HOSTNAME=${OPENSHIFT_KC_HOSTNAME}" >/dev/null + info "Setting Keycloak KC_HOSTNAME=${OPENSHIFT_KC_HOSTNAME} and console redirect host ${console_host}" + # One strategic-merge patch so Keycloak rolls once. HYPERSHELL_CONSOLE_HOST is + # consumed by the render-realm-config init container: --import-realm after a + # pod recycle would otherwise restore localhost-only frontend redirect URIs + # and Keycloak would reject the BFF callback ("Invalid parameter: redirect_uri"). + # `oc set env` without -c only patches spec.containers. A full-object replace + # races Deployment status updates ("the object has been modified"). + keycloak_route_env_patch "${OPENSHIFT_KC_HOSTNAME}" "${console_host}" \ + | oc_cli patch deployment/keycloak -n "${OPENSHIFT_KEYCLOAK_NAMESPACE}" \ + --type=strategic --patch-file=/dev/stdin >/dev/null info "Configuring web console OIDC" oc_cli set env deployment/hypershell-web-console -n "${OPENSHIFT_NAMESPACE}" -c web-console \ @@ -915,6 +950,21 @@ configure_oidc_from_routes() { oc_cli set env deployment/hypershell-web-console -n "${OPENSHIFT_NAMESPACE}" -c web-console \ --from=secret/hypershell-oidc-session >/dev/null + if github_idp_enabled; then + local github_org github_allowlist + github_org="$(oc_cli get secret hypershell-github-oauth -n "${OPENSHIFT_KEYCLOAK_NAMESPACE}" \ + -o jsonpath='{.data.org}' 2>/dev/null | base64 -d 2>/dev/null || true)" + github_allowlist="$(oc_cli get secret hypershell-github-oauth -n "${OPENSHIFT_KEYCLOAK_NAMESPACE}" \ + -o jsonpath='{.data.allowlist}' 2>/dev/null | base64 -d 2>/dev/null || true)" + info "Configuring web console GitHub org gate (${github_org:-openshift-online})" + oc_cli set env deployment/hypershell-web-console -n "${OPENSHIFT_NAMESPACE}" -c web-console \ + "GITHUB_ORG_GATE=${github_org:-openshift-online}" \ + "GITHUB_USERNAME_ALLOWLIST=${github_allowlist}" >/dev/null + else + oc_cli set env deployment/hypershell-web-console -n "${OPENSHIFT_NAMESPACE}" -c web-console \ + GITHUB_ORG_GATE- GITHUB_USERNAME_ALLOWLIST- >/dev/null + fi + info "Configuring control plane public gateway issuer" oc_cli set env deployment/hypershell-controller -n "${OPENSHIFT_NAMESPACE}" -c controller \ "GATEWAY_OIDC_ISSUER_URL=${OPENSHIFT_OIDC_ISSUER}" >/dev/null @@ -1027,11 +1077,28 @@ seed_via_api() { fi return 0 fi + local -a token_args + if github_idp_enabled; then + # Brokered environments have no password grant (ephemeral-pr-environments + # .spec.md); seed as the hypershell-e2e service account instead, which + # holds platform:admin + gateway:creator for exactly this purpose. + local e2e_secret + e2e_secret="$(hypershell_e2e_client_secret)" + if [[ -z "${e2e_secret}" ]]; then + warn "GitHub IDP is enabled but hypershell-github-oauth has no e2e-client-secret; skip automatic seeding" + if seed_strict; then + error "Platform seeding failed and SEED_STRICT=true - failing" + return 1 + fi + return 0 + fi + token_args=(-d grant_type=client_credentials -d client_id=hypershell-e2e -d "client_secret=${e2e_secret}") + else + token_args=(-d grant_type=password -d client_id=hypershell-frontend -d username=admin -d password=admin) + fi info "Obtaining API token from Keycloak Route..." for i in $(seq 1 30); do - resp="$(openshift_curl -X POST "${kc_token_url}" \ - -d grant_type=password -d client_id=hypershell-frontend \ - -d username=admin -d password=admin || true)" + resp="$(openshift_curl -X POST "${kc_token_url}" "${token_args[@]}" || true)" token="$(printf '%s' "${resp}" | json_string_field access_token || true)" if [[ -n "${token}" ]]; then break @@ -1061,12 +1128,6 @@ seed_via_api() { fi } - extract_named_id() { - local resp="$1" name="$2" - printf '%s' "${resp}" | grep -o "\"name\":\"${name}\"[^}]*\"id\":\"[^\"]*\"" \ - | grep -o '"id":"[^"]*"' | cut -d'"' -f4 | head -1 || true - } - extract_id() { local resp="$1" if echo "${resp}" | grep -q '"kind":"Error"'; then @@ -1083,7 +1144,7 @@ seed_via_api() { http="$(printf '%s' "${raw}" | tail -1)" body="$(printf '%s' "${raw}" | sed '$d')" if [[ "${http}" == "200" ]]; then - CLUSTER_ID="$(extract_named_id "${body}" local-openshift)" + CLUSTER_ID="$(printf '%s' "${body}" | json_named_id local-openshift)" fi if [[ -z "${CLUSTER_ID}" ]]; then info "Creating ManagedCluster..." @@ -1107,7 +1168,7 @@ seed_via_api() { http="$(printf '%s' "${raw}" | tail -1)" body="$(printf '%s' "${raw}" | sed '$d')" if [[ "${http}" == "200" ]]; then - RELEASE_ID="$(extract_named_id "${body}" dev-release)" + RELEASE_ID="$(printf '%s' "${body}" | json_named_id dev-release)" fi if [[ -z "${RELEASE_ID}" ]]; then info "Creating GatewayRelease..." @@ -1134,7 +1195,7 @@ seed_via_api() { http="$(printf '%s' "${raw}" | tail -1)" body="$(printf '%s' "${raw}" | sed '$d')" if [[ "${http}" == "200" ]]; then - DATABASE_ID="$(extract_named_id "${body}" openshell-db)" + DATABASE_ID="$(printf '%s' "${body}" | json_named_id openshell-db)" fi if [[ -n "${DATABASE_ID}" ]] && ! cnpg_available \ && printf '%s' "${body}" | grep -Fq '"provider":"cnpg"'; then @@ -1165,24 +1226,36 @@ seed_via_api() { http="$(printf '%s' "${raw}" | tail -1)" body="$(printf '%s' "${raw}" | sed '$d')" if [[ "${http}" == "200" ]]; then - GATEWAY_ID="$(extract_named_id "${body}" dev-gateway)" + GATEWAY_ID="$(printf '%s' "${body}" | json_named_id dev-gateway)" fi + # Keycloak runs start-dev on in-memory H2 with no persistent volume, so any + # Keycloak pod restart (a config change, node drain, upgrade) discards every + # dynamically-provisioned per-gateway OIDC client while dev-gateway's row + # survives untouched in PostgreSQL. Reusing that stale dev-gateway then + # permanently sticks it in status "Keycloak client is missing": the + # reconciler deliberately never auto-recreates a missing client, since doing + # so without also restoring RoleBindings and console mappers would leave it + # silently half-provisioned (openshell-gateway-keycloak.spec.md, "Existing + # gateway client is missing"). Until Keycloak has durable storage, always + # recreate dev-gateway here instead of reusing one that might predate the + # current Keycloak instance. + if [[ -n "${GATEWAY_ID}" ]]; then + info "Recreating dev-gateway ${GATEWAY_ID} (Keycloak has no persistent storage across restarts)..." + api_exec DELETE "/api/hypershell/v1/gateways/${GATEWAY_ID}" >/dev/null + GATEWAY_ID="" + fi + info "Creating Gateway with OIDC..." + local oidc + oidc="{\\\"issuer\\\":\\\"${OPENSHIFT_OIDC_ISSUER}\\\",\\\"audience\\\":\\\"hypershell-frontend\\\",\\\"roles_claim\\\":\\\"groups\\\",\\\"admin_role\\\":\\\"hypershell-admins\\\",\\\"user_role\\\":\\\"hypershell-users\\\"}" + raw="$(api_exec POST /api/hypershell/v1/gateways \ + "{\"name\":\"dev-gateway\",\"cluster_id\":\"${CLUSTER_ID}\",\"release_id\":\"${RELEASE_ID}\",\"database_id\":\"${DATABASE_ID}\",\"oidc\":\"${oidc}\",\"route\":\"{\\\"enabled\\\":true}\"}")" + http="$(printf '%s' "${raw}" | tail -1)" + body="$(printf '%s' "${raw}" | sed '$d')" + GATEWAY_ID="$(extract_id "${body}")" if [[ -z "${GATEWAY_ID}" ]]; then - info "Creating Gateway with OIDC..." - local oidc - oidc="{\\\"issuer\\\":\\\"${OPENSHIFT_OIDC_ISSUER}\\\",\\\"audience\\\":\\\"hypershell-frontend\\\",\\\"roles_claim\\\":\\\"groups\\\",\\\"admin_role\\\":\\\"hypershell-admins\\\",\\\"user_role\\\":\\\"hypershell-users\\\"}" - raw="$(api_exec POST /api/hypershell/v1/gateways \ - "{\"name\":\"dev-gateway\",\"cluster_id\":\"${CLUSTER_ID}\",\"release_id\":\"${RELEASE_ID}\",\"database_id\":\"${DATABASE_ID}\",\"oidc\":\"${oidc}\",\"route\":\"{\\\"enabled\\\":true}\"}")" - http="$(printf '%s' "${raw}" | tail -1)" - body="$(printf '%s' "${raw}" | sed '$d')" - GATEWAY_ID="$(extract_id "${body}")" - if [[ -z "${GATEWAY_ID}" ]]; then - warn "Gateway creation failed (HTTP ${http}): ${body:-no response}" - else - success "Gateway created: ${GATEWAY_ID}" - fi + warn "Gateway creation failed (HTTP ${http}): ${body:-no response}" else - success "dev-gateway already exists: ${GATEWAY_ID}" + success "Gateway created: ${GATEWAY_ID}" fi fi @@ -1206,10 +1279,14 @@ print_banner() { info "Namespace: ${OPENSHIFT_NAMESPACE} (Keycloak: ${OPENSHIFT_KEYCLOAK_NAMESPACE})" info "HTTP API: https://${OPENSHIFT_API_HOST}" info "Web Console: https://${OPENSHIFT_CONSOLE_HOST}" - info "Keycloak: ${OPENSHIFT_KC_HOSTNAME} (admin/admin)" + info "Keycloak: ${OPENSHIFT_KC_HOSTNAME}" info "OIDC Issuer: ${OPENSHIFT_OIDC_ISSUER}" info "Login: https://${OPENSHIFT_CONSOLE_HOST}/auth/login" - info "Test users: admin/admin (admins + users), developer/developer (users only)" + if github_idp_enabled; then + info "Interactive login is GitHub-brokered (openshift-online org, or allowlisted user)" + else + info "Test users: admin/admin (admins + users), developer/developer (users only)" + fi echo "" info "API Server Logs: oc logs -f -l app=hypershell-api-server -n ${OPENSHIFT_NAMESPACE}" info "Control Plane Logs: oc logs -f -l app=hypershell-controller -n ${OPENSHIFT_NAMESPACE}" @@ -1361,6 +1438,53 @@ remove_project() { return 1 } +# Selector for namespaces this control-plane instance stamped. Empty instance is +# refused by the caller; an empty label value would match unlabeled leftovers. +instance_managed_namespace_selector() { + local instance="$1" + printf '%s=%s,%s=%s,%s=%s' \ + "${CP_MANAGED_LABEL}" "${CP_MANAGED_VALUE}" \ + "${MANAGED_LABEL}" "${CP_MANAGED_BY_VALUE}" \ + "${CP_INSTANCE_LABEL}" "${instance}" +} + +# Delete gateway and ManagedDatabase namespaces this instance created. Periodic +# GC cannot do this after the platform project is gone. Never delete the +# platform or keycloak projects through this selector. +delete_instance_managed_namespaces() { + local instance="$1" + if [[ -z "${instance}" ]]; then + error "Refusing to delete instance-managed namespaces with an empty instance identity" + return 1 + fi + info "Removing gateway and database namespaces for instance ${instance}" + local selector names ns failed="" + selector="$(instance_managed_namespace_selector "${instance}")" + names="$(oc_cli get namespace -l "${selector}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)" + if [[ -z "${names}" ]]; then + info "No instance-managed namespaces for ${instance}" + return 0 + fi + while IFS= read -r ns; do + [[ -n "${ns}" ]] || continue + if [[ "${ns}" == "${instance}" || "${ns}" == "${instance}-keycloak" ]]; then + continue + fi + info "Deleting instance-managed namespace ${ns}" + if oc_cli delete namespace "${ns}" --ignore-not-found --wait=true --timeout=300s >/dev/null; then + success "Namespace ${ns} deleted" + else + warn "Failed to delete namespace ${ns}" + failed=true + fi + done <<< "${names}" + if [[ -n "${failed}" ]]; then + error "Failed to delete one or more instance-managed namespaces for ${instance}" + return 1 + fi +} + cluster_down() { header "Removing OpenShift environment" require_openshift_cluster @@ -1377,9 +1501,7 @@ cluster_down() { ok_keycloak=true fi if [[ "${ok_platform}" != "true" && "${ok_keycloak}" != "true" ]]; then - warn "No namespace group found for ${OPENSHIFT_NAMESPACE} / ${OPENSHIFT_KEYCLOAK_NAMESPACE}" - clear_all_openshift_swaps - return 0 + info "No namespace group found for ${OPENSHIFT_NAMESPACE} / ${OPENSHIFT_KEYCLOAK_NAMESPACE}; still reaping instance-managed leftovers" fi info "Deleting this environment's cluster-scoped RBAC..." @@ -1389,9 +1511,14 @@ cluster_down() { oc_cli delete clusterrole "${prefix}hypershell-controller-scc-bind" --ignore-not-found >/dev/null 2>&1 || true oc_cli delete clusterrole "${prefix}hypershell-controller" --ignore-not-found >/dev/null 2>&1 || true - info "Removing namespace group ${OPENSHIFT_NAMESPACE} and ${OPENSHIFT_KEYCLOAK_NAMESPACE}" - remove_project "${OPENSHIFT_KEYCLOAK_NAMESPACE}" - remove_project "${OPENSHIFT_NAMESPACE}" + if [[ "${ok_keycloak}" == "true" || "${ok_platform}" == "true" ]]; then + info "Removing namespace group ${OPENSHIFT_NAMESPACE} and ${OPENSHIFT_KEYCLOAK_NAMESPACE}" + remove_project "${OPENSHIFT_KEYCLOAK_NAMESPACE}" + remove_project "${OPENSHIFT_NAMESPACE}" + fi + # After the controller is gone (or when the platform project was already + # absent) delete sibling gateway/database namespaces this instance stamped. + delete_instance_managed_namespaces "${OPENSHIFT_NAMESPACE}" clear_all_openshift_swaps success "Environment ${OPENSHIFT_NAMESPACE} (and ${OPENSHIFT_KEYCLOAK_NAMESPACE}) removed" } @@ -1508,12 +1635,14 @@ push_component_image() { local repo repo="$(swap_image_repository "${component}")" || exit 1 local push_ref="${repo}:${tag}" - local target_arch + local build_arch target_arch + build_arch="$(swap_build_goarch)" || exit 1 target_arch="$(swap_target_goarch)" || exit 1 - info "Building ${component} from working tree for linux/${target_arch}..." + info "Building ${component} from working tree for linux/${target_arch} (native compile linux/${build_arch})..." ${CONTAINER_ENGINE} build --platform "linux/${target_arch}" -t "${LOCAL_IMAGE}" \ -f "${REPO_ROOT}/${DOCKERFILE}" ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} \ + --build-arg "BUILDARCH=${build_arch}" \ --build-arg "TARGETARCH=${target_arch}" \ --build-arg "TARGETOS=linux" \ "${REPO_ROOT}/${BUILD_CONTEXT}" diff --git a/scripts/cluster/lib.sh b/scripts/cluster/lib.sh index c702cea42..00395fbf7 100755 --- a/scripts/cluster/lib.sh +++ b/scripts/cluster/lib.sh @@ -135,6 +135,19 @@ require_swap_registry() { fi } +# Laptop GOARCH for native compile stages (BUILDARCH). Distinct from +# swap_target_goarch, which is the cluster architecture (TARGETARCH). +swap_build_goarch() { + case "$(uname -m)" in + x86_64) printf 'amd64' ;; + aarch64|arm64) printf 'arm64' ;; + *) + error "Unsupported laptop architecture '$(uname -m)'. Swap builds support amd64 and arm64." + return 1 + ;; + esac +} + # GOARCH for OpenShift swap images. Laptop architecture is not used: ROSA and # most OpenShift nodes are amd64, while developer laptops may be arm64. # SWAP_PLATFORM (linux/amd64 or linux/arm64) wins; otherwise the first node's @@ -448,6 +461,29 @@ print(docs[0]["id"] if docs else "") ' } +# Id of the first HyperShell list item whose name matches. Field order in the +# list payload is not stable (presenters emit id before name), so callers must +# not grep "name" then "id" in one object. Empty on missing name or bad JSON. +json_named_id() { + python3 -c 'import json,sys +name=sys.argv[1] +try: + data=json.load(sys.stdin) +except Exception: + sys.exit(0) +if isinstance(data, dict): + items=data.get("items") or [] +elif isinstance(data, list): + items=data +else: + items=[] +for it in items: + if isinstance(it, dict) and it.get("name") == name: + print(it.get("id") or "") + break +' "$1" +} + # Restrict a Keycloak client representation to this console origin. # Spec: oidc-integration Identity Provider Client Security: no wildcards. keycloak_client_with_console_redirects() { @@ -460,6 +496,36 @@ json.dump(doc, sys.stdout) ' "${console_host}" } +# Strategic-merge patch that stamps Route-derived Keycloak env on the named +# containers. HYPERSHELL_CONSOLE_HOST is read by init container +# render-realm-config, not by the keycloak container. `oc set env` without -c +# only patches spec.containers on OpenShift, so a later start-dev --import-realm +# on empty H2 would restore localhost frontend redirect URIs. A full-object +# replace races Deployment status updates ("the object has been modified"). +keycloak_route_env_patch() { + local kc_hostname="$1" + local console_host="$2" + python3 -c 'import json,sys +kc_hostname, console_host = sys.argv[1], sys.argv[2] +json.dump({ + "spec": { + "template": { + "spec": { + "initContainers": [{ + "name": "render-realm-config", + "env": [{"name": "HYPERSHELL_CONSOLE_HOST", "value": console_host}], + }], + "containers": [{ + "name": "keycloak", + "env": [{"name": "KC_HOSTNAME", "value": kc_hostname}], + }], + } + } + } +}, sys.stdout) +' "${kc_hostname}" "${console_host}" +} + # SKIP_SEED and SEED_STRICT apply to Kind and OpenShift. KIND_* names remain aliases. skip_seed() { case "${SKIP_SEED:-${KIND_SKIP_SEED:-}}" in diff --git a/scripts/cluster/lib_test.sh b/scripts/cluster/lib_test.sh index d6d4a9546..6110aa422 100755 --- a/scripts/cluster/lib_test.sh +++ b/scripts/cluster/lib_test.sh @@ -67,6 +67,27 @@ assert_fail "uppercase rejected" validate_rfc1123_label "Alice" 54 assert_fail "underscore rejected" validate_rfc1123_label "alice_dev" 54 assert_eq "tok" "$(printf '%s' '{"access_token":"tok","expires_in":60}' | json_string_field access_token)" "json_string_field access_token" assert_eq "abc-id" "$(printf '%s' '[{"id":"abc-id","clientId":"hypershell-frontend"}]' | json_first_id)" "json_first_id" +# Presenters emit id before name. The old grep ("name" then "id" in one object) +# misses this shape and re-POSTs a second dev-gateway on every openshift-seed. +_api_list='{"kind":"GatewayList","page":1,"size":100,"total":1,"items":[{"id":"2FhMpQzXBzABC","kind":"Gateway","href":"/api/hypershell/v1/gateways/2FhMpQzXBzABC","created_at":"2026-09-14T00:00:00Z","updated_at":"2026-09-14T00:00:00Z","name":"dev-gateway","cluster_id":"c1","release_id":"r1"}]}' +assert_eq "2FhMpQzXBzABC" "$(printf '%s' "${_api_list}" | json_named_id dev-gateway)" \ + "json_named_id finds id-before-name list items" +assert_eq "id-default" "$(printf '%s' '{"items":[{"name":"other","id":"id-other"},{"name":"dev-gateway","id":"id-default"}]}' | json_named_id dev-gateway)" \ + "json_named_id finds name-before-id list items" +assert_eq "" "$(printf '%s' "${_api_list}" | json_named_id missing-gateway)" \ + "json_named_id is empty when the name is absent" +assert_eq "" "$(printf '%s' 'not-json' | json_named_id dev-gateway)" \ + "json_named_id is empty on invalid JSON" +_pretty_list='{ + "items": [ + { + "id": "pretty-id", + "name": "dev-gateway" + } + ] +}' +assert_eq "pretty-id" "$(printf '%s' "${_pretty_list}" | json_named_id dev-gateway)" \ + "json_named_id finds pretty-printed list items" assert_ok "internal registry svc:port is cluster-local" \ registry_host_is_cluster_local 'image-registry.openshift-image-registry.svc:5000' assert_ok "cluster.local registry is cluster-local" \ @@ -101,6 +122,14 @@ assert_eq "amd64" "$(SWAP_PLATFORM=linux/amd64 swap_target_goarch)" "SWAP_PLATFO assert_eq "amd64" "$(SWAP_ARCH=x86_64 SWAP_PLATFORM= swap_target_goarch)" "SWAP_ARCH x86_64" assert_eq "arm64" "$(SWAP_PLATFORM=linux/arm64 swap_target_goarch)" "SWAP_PLATFORM linux/arm64" SWAP_PLATFORM=linux/ppc64le assert_fail "unsupported SWAP_PLATFORM" swap_target_goarch +case "$(uname -m)" in + x86_64) _expected_build_arch=amd64 ;; + aarch64|arm64) _expected_build_arch=arm64 ;; + *) _expected_build_arch="" ;; +esac +if [[ -n "${_expected_build_arch}" ]]; then + assert_eq "${_expected_build_arch}" "$(swap_build_goarch)" "laptop BUILDARCH matches uname -m" +fi unset SWAP_PLATFORM SWAP_ARCH assert_eq "sha256:d3f6ac0a7627fee89b55f34745e09fc64d0073e807719a66f6b4534a96541eb6" \ "$(printf '%s\n' \ @@ -148,6 +177,52 @@ merged="$(printf '%s' '{"id":"x","redirectUris":["https://console.hypershell.loc assert_eq '["https://console.apps.example.com/auth/callback", "https://console.apps.example.com"]' \ "$(printf '%s' "${merged}" | python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin)["redirectUris"]))')" \ "keycloak_client_with_console_redirects replaces Kind localhost URIs" +_kc_patch="$(keycloak_route_env_patch 'https://keycloak.apps.example.com' 'web-console.apps.example.com')" +assert_eq 'https://keycloak.apps.example.com' \ + "$(printf '%s' "${_kc_patch}" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["spec"]["template"]["spec"]["containers"][0]["env"][0]["value"])')" \ + "keycloak_route_env_patch sets KC_HOSTNAME on the keycloak container" +assert_eq 'keycloak' \ + "$(printf '%s' "${_kc_patch}" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["spec"]["template"]["spec"]["containers"][0]["name"])')" \ + "keycloak_route_env_patch targets container keycloak" +assert_eq 'web-console.apps.example.com' \ + "$(printf '%s' "${_kc_patch}" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["spec"]["template"]["spec"]["initContainers"][0]["env"][0]["value"])')" \ + "keycloak_route_env_patch sets HYPERSHELL_CONSOLE_HOST on render-realm-config" +assert_eq 'render-realm-config' \ + "$(printf '%s' "${_kc_patch}" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["spec"]["template"]["spec"]["initContainers"][0]["name"])')" \ + "keycloak_route_env_patch targets init container render-realm-config" +assert_eq 'HYPERSHELL_CONSOLE_HOST' \ + "$(printf '%s' "${_kc_patch}" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["spec"]["template"]["spec"]["initContainers"][0]["env"][0]["name"])')" \ + "keycloak_route_env_patch names the console-host env var" +if grep -A14 'Setting Keycloak KC_HOSTNAME' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'keycloak_route_env_patch'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: openshift-up does not stamp Keycloak route env via keycloak_route_env_patch' +fi +if grep -A14 'Setting Keycloak KC_HOSTNAME' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q -- '--type=strategic'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: openshift-up does not strategic-merge patch Keycloak route env' +fi +if grep -A14 'Setting Keycloak KC_HOSTNAME' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -Eq 'replace -f|set env deployment/keycloak'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: openshift-up still replaces or set-envs the Keycloak Deployment' +else + PASS=$((PASS + 1)) +fi +if grep -A25 'Recycle Keycloak when GitHub OAuth secret changes' "${REPO_ROOT}/.github/workflows/pr-environment.yml" | grep -q 'already matches; skip recycle'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: PR env workflow recycles Keycloak even when the oauth-secret hash is unchanged' +fi +if grep 'Keycloak:' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'admin/admin'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: Keycloak banner still prints admin/admin outside the test-user branch' +else + PASS=$((PASS + 1)) +fi if grep -A3 '^cluster_teardown()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'cluster_down'; then PASS=$((PASS + 1)) else @@ -212,6 +287,35 @@ else FAIL=$((FAIL + 1)) echo 'FAIL: OpenShift cluster_down does not use the -dev- cluster-scoped prefix' fi +if grep -A80 '^cluster_down()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'delete_instance_managed_namespaces'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift cluster_down does not delete instance-managed gateway namespaces' +fi +if grep -A20 '^delete_instance_managed_namespaces()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'empty instance identity'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: delete_instance_managed_namespaces does not refuse an empty instance' +fi +_selector_got="$(bash -c ' + # shellcheck source=lib.sh + source "'"${SCRIPT_DIR}"'/lib.sh" + # shellcheck source=drivers/openshift.sh + source "'"${SCRIPT_DIR}"'/drivers/openshift.sh" + instance_managed_namespace_selector alice +')" +assert_eq \ + 'hypershell.redhat.io/managed=true,app.kubernetes.io/managed-by=hypershell-control-plane,hypershell.redhat.io/instance=alice' \ + "${_selector_got}" \ + "instance selector stamps managed + managed-by + instance" +if grep -A80 '^cluster_down()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'still reaping instance-managed leftovers'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift cluster_down returns early when the platform project is already gone' +fi if grep -E 'delete clusterrole(binding)? "hypershell-controller' "${SCRIPT_DIR}/drivers/openshift.sh"; then FAIL=$((FAIL + 1)) echo 'FAIL: OpenShift down deletes unprefixed cluster-scoped names (would hit stage)' @@ -256,6 +360,35 @@ else FAIL=$((FAIL + 1)) echo 'FAIL: OpenShift cluster_up does not honor SKIP_SEED' fi +if grep -q 'extract_named_id' "${SCRIPT_DIR}/drivers/openshift.sh" \ + || grep -Fq '[^}]*"id"' "${SCRIPT_DIR}/drivers/openshift.sh"; then + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift seed still greps name-then-id (misses API list JSON)' +else + PASS=$((PASS + 1)) +fi +if grep -q 'json_named_id local-openshift' "${SCRIPT_DIR}/drivers/openshift.sh" \ + && grep -q 'json_named_id dev-release' "${SCRIPT_DIR}/drivers/openshift.sh" \ + && grep -q 'json_named_id openshell-db' "${SCRIPT_DIR}/drivers/openshift.sh" \ + && grep -q 'json_named_id dev-gateway' "${SCRIPT_DIR}/drivers/openshift.sh"; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift seed_via_api does not look up existing resources with json_named_id' +fi +if grep -Fq '[^}]*"id"' "${REPO_ROOT}/scripts/kind/seed.sh"; then + FAIL=$((FAIL + 1)) + echo 'FAIL: Kind seed still greps name-then-id (misses API list JSON)' +else + PASS=$((PASS + 1)) +fi +if grep -q 'json_named_id local-kind' "${REPO_ROOT}/scripts/kind/seed.sh" \ + && grep -q 'json_named_id dev-gateway' "${REPO_ROOT}/scripts/kind/seed.sh"; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: Kind seed does not look up existing resources with json_named_id' +fi if grep -B2 'db_provider="\$(effective_database_provider)"' "${SCRIPT_DIR}/drivers/openshift.sh" >/dev/null \ && grep -A20 'Creating ManagedDatabase' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'provider='; then PASS=$((PASS + 1)) @@ -940,6 +1073,13 @@ else FAIL=$((FAIL + 1)) echo 'FAIL: swap build does not pass TARGETARCH for the cluster node architecture' fi +if grep -A80 '^push_component_image()' "${SCRIPT_DIR}/drivers/openshift.sh" \ + | grep -q 'BUILDARCH'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: swap build does not pass BUILDARCH for the laptop architecture' +fi if grep -A80 '^push_component_image()' "${SCRIPT_DIR}/drivers/openshift.sh" \ | grep -q -- '--platform'; then PASS=$((PASS + 1)) @@ -969,6 +1109,14 @@ else FAIL=$((FAIL + 1)) echo 'FAIL: Go Dockerfiles do not honor TARGETARCH for OpenShift swap cross-compile' fi +if grep -q 'build-${BUILDARCH}' "${REPO_ROOT}/components/web-console/Dockerfile" \ + && grep -q 'AS bundle' "${REPO_ROOT}/components/web-console/Dockerfile" \ + && grep -q 'build-${TARGETARCH}' "${REPO_ROOT}/components/web-console/Dockerfile"; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: web-console Dockerfile does not compile on BUILDARCH and bundle native addons on TARGETARCH' +fi if grep -q 'OPENSHIFT_IMAGE_REGISTRY' "${SCRIPT_DIR}/drivers/openshift.sh" "${SCRIPT_DIR}/lib.sh" "${REPO_ROOT}/Makefile"; then FAIL=$((FAIL + 1)) echo 'FAIL: OPENSHIFT_IMAGE_REGISTRY is still present; swaps use SWAP_REGISTRY' @@ -993,6 +1141,26 @@ else FAIL=$((FAIL + 1)) echo 'FAIL: kind-up does not accept PULL_SECRET with KIND_PULL_SECRET alias' fi +# Keycloak has no persistent storage (start-dev on in-memory H2), so a Keycloak +# pod restart discards dev-gateway's OIDC client while its row survives in +# PostgreSQL, permanently sticking it in "Keycloak client is missing" (the +# reconciler never auto-recreates a missing client). Until Keycloak gets +# durable storage, seed_via_api must delete and recreate dev-gateway on every +# run instead of reusing whatever it finds. +if awk '/^seed_via_api\(\)/,0' "${SCRIPT_DIR}/drivers/openshift.sh" \ + | grep -A20 'json_named_id dev-gateway' | grep -q 'api_exec DELETE "/api/hypershell/v1/gateways/\${GATEWAY_ID}"'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift seed_via_api does not delete an existing dev-gateway before recreating it' +fi +if awk '/^seed_via_api\(\)/,0' "${SCRIPT_DIR}/drivers/openshift.sh" \ + | grep -q 'dev-gateway already exists'; then + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift seed_via_api still reuses an existing dev-gateway instead of recreating it' +else + PASS=$((PASS + 1)) +fi printf 'OpenShift lifecycle tests: %d passed, %d failed\n' "${PASS}" "${FAIL}" [[ "${FAIL}" -eq 0 ]] diff --git a/scripts/kind/lib.sh b/scripts/kind/lib.sh index 19dd08675..78adc4807 100755 --- a/scripts/kind/lib.sh +++ b/scripts/kind/lib.sh @@ -59,6 +59,29 @@ seed_strict() { esac } +# Id of the first HyperShell list item whose name matches. Field order in the +# list payload is not stable (presenters emit id before name), so callers must +# not grep "name" then "id" in one object. Empty on missing name or bad JSON. +json_named_id() { + python3 -c 'import json,sys +name=sys.argv[1] +try: + data=json.load(sys.stdin) +except Exception: + sys.exit(0) +if isinstance(data, dict): + items=data.get("items") or [] +elif isinstance(data, list): + items=data +else: + items=[] +for it in items: + if isinstance(it, dict) and it.get("name") == name: + print(it.get("id") or "") + break +' "$1" +} + # --- Cluster helpers --- cluster_exists() { diff --git a/scripts/kind/lib_test.sh b/scripts/kind/lib_test.sh index 624481475..f68de50f0 100755 --- a/scripts/kind/lib_test.sh +++ b/scripts/kind/lib_test.sh @@ -71,6 +71,12 @@ assert_ok "tab-format web-console is swapped" is_swapped web-console assert_eq "hot-reload" "$(swap_image web-console)" "tab-format swap_image" assert_fail "api-server-extra is not matched as api-server" is_swapped api-server-extra +_api_list='{"kind":"GatewayList","items":[{"id":"2FhMpQzXBzABC","kind":"Gateway","name":"dev-gateway","cluster_id":"c1"}]}' +assert_eq "2FhMpQzXBzABC" "$(printf '%s' "${_api_list}" | json_named_id dev-gateway)" \ + "json_named_id finds id-before-name list items" +assert_eq "" "$(printf '%s' "${_api_list}" | json_named_id missing-gateway)" \ + "json_named_id is empty when the name is absent" + echo "Kind swap ledger tests: ${PASS} passed, ${FAIL} failed" if ((FAIL > 0)); then exit 1 diff --git a/scripts/kind/seed.sh b/scripts/kind/seed.sh index 5b6f050e4..c68337d12 100755 --- a/scripts/kind/seed.sh +++ b/scripts/kind/seed.sh @@ -158,7 +158,7 @@ if [[ -z "${seed_failed}" ]]; then EXISTING_MC_RESP=$(echo "${EXISTING_MC_RAW}" | sed '$d') if [[ "${EXISTING_MC_HTTP}" == "200" ]]; then - CLUSTER_ID=$(echo "${EXISTING_MC_RESP}" | grep -o '"name":"local-kind"[^}]*"id":"[^"]*"' | grep -o '"id":"[^"]*"' | cut -d'"' -f4 | head -1 || true) + CLUSTER_ID=$(printf '%s' "${EXISTING_MC_RESP}" | json_named_id local-kind) if [[ -n "${CLUSTER_ID}" ]]; then success "local-kind ManagedCluster already exists: ${CLUSTER_ID}" fi @@ -193,7 +193,7 @@ if [[ -z "${seed_failed}" ]]; then EXISTING_GR_RESP=$(echo "${EXISTING_GR_RAW}" | sed '$d') if [[ "${EXISTING_GR_HTTP}" == "200" ]]; then - RELEASE_ID=$(echo "${EXISTING_GR_RESP}" | grep -o '"name":"dev-release"[^}]*"id":"[^"]*"' | grep -o '"id":"[^"]*"' | cut -d'"' -f4 | head -1 || true) + RELEASE_ID=$(printf '%s' "${EXISTING_GR_RESP}" | json_named_id dev-release) if [[ -n "${RELEASE_ID}" ]]; then success "dev-release GatewayRelease already exists: ${RELEASE_ID}" fi @@ -228,7 +228,7 @@ if [[ -z "${seed_failed}" ]]; then EXISTING_MD_RESP=$(echo "${EXISTING_MD_RAW}" | sed '$d') if [[ "${EXISTING_MD_HTTP}" == "200" ]]; then - DATABASE_ID=$(echo "${EXISTING_MD_RESP}" | grep -o '"name":"openshell-db"[^}]*"id":"[^"]*"' | grep -o '"id":"[^"]*"' | cut -d'"' -f4 | head -1 || true) + DATABASE_ID=$(printf '%s' "${EXISTING_MD_RESP}" | json_named_id openshell-db) if [[ -n "${DATABASE_ID}" ]]; then success "openshell-db ManagedDatabase already exists: ${DATABASE_ID}" fi @@ -274,7 +274,7 @@ if [[ -z "${seed_failed}" ]]; then EXISTING_GW_RESP=$(echo "${EXISTING_GW_RAW}" | sed '$d') if [[ "${EXISTING_GW_HTTP}" == "200" ]]; then - EXISTING_GW_ID=$(echo "${EXISTING_GW_RESP}" | grep -o '"name":"dev-gateway"[^}]*"id":"[^"]*"' | grep -o '"id":"[^"]*"' | cut -d'"' -f4 | head -1 || true) + EXISTING_GW_ID=$(printf '%s' "${EXISTING_GW_RESP}" | json_named_id dev-gateway) if [[ -n "${EXISTING_GW_ID}" ]]; then success "dev-gateway already exists: ${EXISTING_GW_ID}" GATEWAY_ID="${EXISTING_GW_ID}" diff --git a/skills/RECONCILE.md b/skills/RECONCILE.md index 86a030e8d..8a1cfce92 100644 --- a/skills/RECONCILE.md +++ b/skills/RECONCILE.md @@ -49,7 +49,7 @@ skills/ ## Reconciliation State -**Last analyzed**: 2026-09-14 (scoped analysis of specs/platform/gateway-deletion-finalization.spec.md for HYPERSHELL-182; closed the no-silent-orphan gap G1: best-effort deletion failures for gateway-owned resources with no automatic recovery path -- leaked ClusterRoleBinding, leaked Keycloak gateway/console clients, and Keycloak clients skipped when the stored identity is unresolvable or the provisioner is deconfigured -- now emit a durable IncompleteFinalization Warning Event in the control-plane namespace instead of only logging; in-namespace sweep G2 already satisfied; the last full-corpus analysis remains 2026-08-31) +**Last analyzed**: 2026-09-14 (scoped analysis of specs/platform/gateway-deletion-finalization.spec.md for HYPERSHELL-182; closed the no-silent-orphan gap G1: best-effort deletion failures for gateway-owned resources with no automatic recovery path -- leaked ClusterRoleBinding, leaked Keycloak gateway/console clients, and Keycloak clients skipped when the stored identity is unresolvable or the provisioner is deconfigured -- now emit a durable IncompleteFinalization Warning Event in the control-plane namespace instead of only logging; in-namespace sweep G2 already satisfied; the last full-corpus analysis remains 2026-08-31). Prior 2026-09-09 scoped reanalysis of e2e-testing.spec.md + local-development.spec.md against HEAD `21f02a0` for the new OpenShift E2E CI content added by the HYPERSHELL-240 docs commit: OpenShift driver unification #232/#244, dynamic namespace-GC timing, and the merge-queue Kind CI gate are all implemented; the only remaining gap is D-E2E-OIDC, the `E2E_OIDC_GRANT=client_credentials` token-exchange grant whose mechanics belong to `ephemeral-pr-environments.spec.md` and is a divergence pending scope decision. Prior 2026-09-04 scoped reanalysis of the CP-OBS-07 reconcile-queue metric changes after review; operational-dashboard through OP-DASH-20; OP-DASH-18 NaN fallback; OP-DASH-19 independent metric sources + partial failure; OP-DASH-20 section titles + header refresh consolidation; cluster memory/cpu/pods/nodes metrics; gateway-provision-time GPT-W1; registered-users complete; the last full-corpus analysis remains 2026-08-31) **Spec corpus**: 49 spec files; the coverage table tracks 39 analyzed feature/spec groups after adding OpenShell Gateway Console, OpenShift Development, Operational Dashboard, Registered Users, Cluster Memory, Cluster CPU, Cluster Pods, Cluster Nodes, and Gateway Provision Time **Codebase commit**: `608da30` (Update Konflux references; then HYPERSHELL-182 gateway deletion finalization: OrphanRecorder callback on ReconcileOpts + recordIncompleteFinalizationEvent durable Event) @@ -72,6 +72,7 @@ skills/ | Platform - Sandbox Count | 1 | 6 | 6 | 0 | 0 | 0 | 100% | | Platform - Local Development | 1 | 25 | 23 | 0 | 1 | 1 | 96% | | Platform - E2E Testing | 1 | 19 | 19 | 0 | 0 | 0 | 100% | +| Platform - Ephemeral PR Environments | 1 | 11 | 9 | 2 | 0 | 0 | 91% | | Platform - OpenShift Development | 1 | 13 | 7 | 2 | 4 | 0 | 54% | | Platform - OIDC Integration | 1 | 7 | 6 | 1 | 0 | 0 | 93% | | Platform - Gateway Metrics Dashboard | 1 | 8 | 8 | 0 | 0 | 0 | 100% | @@ -577,6 +578,10 @@ Local-dev lifecycle (`make openshift-up` / `down` / component swaps) is implemen | E2E-7 | Deploy Base/Overlay Structure | Present | deploy/base/ + deploy/kind/ overlay + deploy/openshift/ stub | `deploy/base/`, `deploy/kind/kustomization.yaml` | E2E-W1 ✅ | | E2E-8 | Backward Compatibility | Present | make kind-up unchanged; IMAGE_TAG now overrides initial deploy images | `scripts/kind/up.sh` | E2E-W1 ✅ | | E2E-9 | E2E short and long modes (`E2E_MODE`) | Present | Step-tagged `e2e_step short\|long`; default `long`; invalid mode fails fast | `tests/e2e/lib.sh`, `tests/e2e/e2e-openshell.sh` | PERF-W1 ✅ | +| E2E-10 | OpenShift e2e driver (contract parity) | Present | Delivered by #232/#244: OpenShift driver unified with Kind (shared token/role helpers, Route discovery, shared-Gateway base domain) | `tests/e2e/drivers/openshift.sh`, `tests/e2e/openshift_driver_test.sh` | HYPERSHELL-44 ✅ | +| E2E-11 | Dynamic namespace GC timing | Present | `configure_namespace_gc_timing` / `restore_namespace_gc_timing` patch controller env + restore on cleanup; no overlay bakes e2e timing | `tests/e2e/drivers/kind.sh`, `tests/e2e/drivers/openshift.sh`, `tests/e2e/e2e-openshell.sh` | #244 ✅ | +| E2E-12 | Merge-queue Kind CI gate | Present | `e2e.yml` `merge_group` trigger always runs; per-component merge-queue Konflux waits on `on-merge-queue-`; browser trace skipped on `merge_group`; dedicated `.tekton/*-merge-queue.yaml` | `.github/workflows/e2e.yml`, `.tekton/hypershell-*-main-merge-queue.yaml` | #161/#232 ✅ | +| D-E2E-OIDC | `E2E_OIDC_GRANT` grant selection (`client_credentials` + token-exchange) | Missing (Divergence) | Driver token fns hardcode `grant_type=password` (`kind.sh:129,370`); no `E2E_OIDC_GRANT` handling. The GitHub-brokered PR mechanics (`hypershell-e2e` client-credentials + token-exchange impersonation) are owned by `ephemeral-pr-environments.spec.md` (HYPERSHELL-240), out of the requested scope; e2e-testing.spec.md says "SHALL NOT be restated here" | `tests/e2e/drivers/kind.sh` | Needs decision (HYPERSHELL-240) | | PERF-1 | `make e2e-performance` entry point | Present | Defaults `E2E_INFRA_DRIVER` to `kind`; honors CLI override | `Makefile` | PERF-W1 ✅ | | PERF-2 | Infra-agnostic harness | Present | Driver-selected; `$(get_cli_binary)` only; no kubectl/oc/kind in harness | `tests/e2e/e2e-performance.sh` | PERF-W1 ✅ | | PERF-3 | Gateway fleet scale-up | Present | Batch + bounded concurrency, reuse-or-create, per-gateway latency | `tests/e2e/e2e-performance.sh`, `tests/e2e/perf/lib.sh` | PERF-W1 ✅ | @@ -588,7 +593,39 @@ Local-dev lifecycle (`make openshift-up` / `down` / component swaps) is implemen | PERF-9 | Performance cleanup | Present | EXIT trap deletes fleet + canary + functional GW; bounded concurrency; skippable | `tests/e2e/e2e-performance.sh` | PERF-W1 ✅ | | PERF-10 | Failure diagnostics | Present | Pending pods, node capacity, gateway phases, CP logs; `::group::` only under Actions | `tests/e2e/perf/lib.sh` | PERF-W1 ✅ | -The OpenShift e2e driver (`tests/e2e/drivers/openshift.sh`) remains a gap for HYPERSHELL-44; this wave delivered the performance harness against the existing Kind driver. `make e2e` / `make e2e-performance` honor `E2E_INFRA_DRIVER=openshift` once that driver exists. +The OpenShift e2e driver (`tests/e2e/drivers/openshift.sh`) now exists and is unified with the Kind driver (#232/#244): it shares the token/role helpers, discovers the API/console via Routes, derives the gateway base domain from the shared Gateway listener, and overrides only where OpenShift constructs differ. Dynamic namespace-GC timing (`configure_namespace_gc_timing` / `restore_namespace_gc_timing`) and the merge-queue Kind CI gate (`merge_group` trigger, dedicated `.tekton/*-merge-queue.yaml`, browser-trace skip) are implemented. `make e2e` / `make e2e-performance` honor `E2E_INFRA_DRIVER=openshift`. + +The single remaining e2e-testing gap after the 2026-09-09 docs update (commit `21f02a0`, HYPERSHELL-240) is **D-E2E-OIDC**: the driver token functions do not yet honor `E2E_OIDC_GRANT=client_credentials` (GitHub-brokered token-exchange path). Its mechanics are specified in `ephemeral-pr-environments.spec.md` (HYPERSHELL-240) and are tracked there as PR-ENV-10. + +### ephemeral-pr-environments.spec.md (HYPERSHELL-240) + +Greenfield: no code references `hypershell-ci-pr-*`, `expires-at`, GitHub IdP brokering, the `hypershell-e2e` client, allowlist, impersonation, or token-exchange. Builds on the existing `make openshift-up` lifecycle (`scripts/cluster/drivers/openshift.sh`), the `deploy/openshift/` overlay, the Keycloak realm JSON ConfigMap (`deploy/base/keycloak/keycloak.yaml`), and `scripts/kind/set-component-images.sh`. + +| # | Requirement | Status | Gap | Code Location | Wave | +|---|-------------|--------|-----|---------------|------| +| PR-ENV-1 | Per-PR environment identity (`hypershell-ci-pr-` + labels) | Present | `pr_env_namespace/environment_id` derive `hypershell-ci-pr-` + `pr-`; `stamp-pr-env.sh` overwrites env id + fails closed on label error; workflow sets `OPENSHIFT_NAMESPACE`. Unit-tested. | `scripts/ci/pr-env-lib.sh`, `scripts/ci/stamp-pr-env.sh`, `.github/workflows/pr-environment.yml` | W3 ✅ | +| PR-ENV-2 | Continuous deployment lifecycle (opened/reopened/synchronize/closed; unconditional `openshift-up`; per-PR concurrency) | Present | Deploy workflow triggers on open/reopen/synchronize; release workflow on `closed` (merge or close); `make openshift-up` unconditional with `SKIP_SEED`; shared `concurrency: pr-env-` cancel-in-progress | `.github/workflows/pr-environment.yml`, `.github/workflows/pr-environment-release.yml` | W3 ✅ | +| PR-ENV-3 | Image gating + swap by digest (Konflux; reuse `set-component-images.sh` mechanism) | Present | `plan-images` mirrors e2e CEL triggers -> `on-pr-`; `wait-on-check-action` gates; `swap-openshift-images-by-digest.sh` resolves `@sha256` (tag fallback recorded) | `.github/workflows/pr-environment.yml`, `scripts/ci/swap-openshift-images-by-digest.sh` | W3 ✅ | +| PR-ENV-4 | E2E against the environment (`E2E_INFRA_DRIVER=openshift E2E_OIDC_GRANT=client_credentials`) | Present | Tests / E2E / OpenShift polls the `Deploy PR environment` check, then runs the shared harness with those envs + `E2E_OIDC_SA_CLIENT_SECRET` read from the Keycloak ns; diagnostics on failure; env survives | `.github/workflows/e2e.yml`, `.github/workflows/tests.yml`, `scripts/ci/read-e2e-client-secret.sh` | W3 ✅ | +| PR-ENV-5 | Timebox (3d `expires-at`) + out-of-band reaper | Present | `stamp-pr-env.sh` stamps `expires-at` (configurable `PR_ENV_TIMEBOX_DAYS`); `reap-pr-environments.sh` + `deploy/e2e/reaper` CronJob deletes expired `pr-*` groups; `close` releases via `openshift-down`. Reaper predicate unit-tested. **Deploy the reaper CronJob to the cluster (manual/Argo).** | `scripts/ci/pr-env-lib.sh`, `scripts/ci/reap-pr-environments.sh`, `deploy/e2e/reaper/`, `.github/workflows/pr-environment.yml` | W4 ✅ | +| PR-ENV-6 | PR comment + access handoff (marked, one-per-PR, no creds) | Present | `pr_env_comment_body` carries the hidden marker + redacted `oc login`; `upsert-pr-comment.sh` finds/updates the marked comment. Marker/redaction unit-tested. | `scripts/ci/pr-env-lib.sh`, `scripts/ci/upsert-pr-comment.sh` | W3 ✅ | +| PR-ENV-7 | Trust boundary (origin-only `pull_request`, no `pull_request_target`, no fork creds) | Present | `pull_request` only; every job guarded on `head.repo.full_name == github.repository` so forks get no secrets/env | `.github/workflows/pr-environment.yml` | W3 ✅ | +| PR-ENV-8 | GitHub-brokered Keycloak (GitHub IdP, org gate + allowlist, stable callback) | Present | **Declarative IdP:** GitHub IdP, org/allowlist realm attributes, and `hypershell-e2e` client secret live in the base realm as placeholders resolved from the optional `hypershell-github-oauth` Secret. Kind/local/stage have no Secret, so the IdP stays off. **Org-gate enforcement (weaker, no custom Keycloak image):** the web-console BFF checks org membership / allowlist after the OIDC callback (`GITHUB_ORG_GATE` / `GITHUB_USERNAME_ALLOWLIST`, copied from the Secret by `openshift-up`). Denied users get no HyperShell session, so the BFF never forwards an API bearer. The API server is not separately org-gated. Unit-tested. | `deploy/base/keycloak/keycloak.yaml`, `components/web-console/bff/src/github-org-gate.ts`, `components/web-console/bff/src/auth.ts`, `scripts/cluster/drivers/openshift.sh`, `.github/workflows/pr-environment.yml` | W2 ✅ | +| PR-ENV-9 | Admin roles + developer-tier impersonation (seeded dev principal) | Partial | Base realm carries `identityProviderMappers` (hardcoded-role) granting `platform:admin`+`gateway:creator` on GitHub broker login; `hypershell-e2e` has standard token exchange (W1). Inert on Kind (IdP disabled). **Handoff:** seeding the `gateway:viewer`/`openshell-user` developer principal + admin impersonation wiring. | `deploy/base/keycloak/keycloak.yaml` | W2 (partial) | +| PR-ENV-10 | Automated E2E auth: grant-agnostic driver (`E2E_OIDC_GRANT`), `hypershell-e2e` client-credentials + token-exchange (= D-E2E-OIDC) | Partial | **Code done (W1):** `_driver_acquire_oidc_token` dispatches password/client_credentials (admin CC + developer token-exchange impersonation); `E2E_OIDC_GRANT`/`E2E_OIDC_SA_CLIENT_ID`/`_SECRET` defaults; `hypershell-e2e` confidential client + SA (platform:admin+gateway:creator, audience mapper, standard token exchange) in base realm. Unit-tested (`openshift_driver_test.sh` 20/20). **Remaining:** CI reads the client secret from the Keycloak namespace (W3 workflow). | `tests/e2e/drivers/kind.sh`, `tests/e2e/lib.sh`, `deploy/base/keycloak/keycloak.yaml`, `tests/e2e/openshift_driver_test.sh` | W1 ✅ | +| PR-ENV-11 | Legacy `pr-test` deprecation notice + docs | Present | Deprecation header on `components/pr-test/e2e-openshell.sh` (names shared harness; excludes ROKS); DEVELOPMENT.md pointer to `tests/e2e/e2e-openshell.sh` + `E2E_INFRA_DRIVER=openshift`; ROKS/GCP variants + `pr_test` CI wiring untouched | `components/pr-test/e2e-openshell.sh`, `DEVELOPMENT.md` | W5 ✅ | + +**Human-provisioned prerequisites (not code):** GitHub OAuth App (client id/secret) + one stable callback URL, CI cluster kubeconfig secret, the `openshift-online` org gate + allowlist values, and deploying the reaper onto the target cluster (or a scheduled runner). + +**Handoff to finish HYPERSHELL-240 (outside this workflow):** +1. **GitHub Actions secrets** (repo or a `pr-environments` environment): `OPENSHIFT_PR_ENV_SERVER_URL` (cluster API URL, kept as a secret so it can be changed without code), `OPENSHIFT_PR_ENV_TOKEN` (CI service-account token for `oc login`), `PR_ENV_GITHUB_OAUTH_CLIENT_ID`, `PR_ENV_GITHUB_OAUTH_CLIENT_SECRET`, `PR_ENV_GITHUB_OAUTH_CALLBACK_URL`. **Vars:** `PR_ENV_GITHUB_ORG` (default `openshift-online`), `PR_ENV_GITHUB_ALLOWLIST` (comma-separated), `PR_ENV_TIMEBOX_DAYS` (default 3). +2. **CI service account on the cluster:** create an SA with rights to create/patch/delete projects, apply the overlay, patch namespaces, and read Secrets in the `-keycloak` namespace; mint its token into `OPENSHIFT_PR_ENV_TOKEN`. +3. **GitHub OAuth App:** register one App with a single stable callback URL (cluster infra, like the shared Gateway), set the three OAuth secrets above. +4. **Deploy the reaper:** `kustomize build --load-restrictor=LoadRestrictionsNone deploy/e2e/reaper | oc apply -f -` (or via Argo). Adjust the `ose-cli` image / schedule as needed. +5. **Kind smoke test (optional):** `make kind-up` + a Keycloak boot to confirm `${PR_ENV_GITHUB_*:default}` / `${HYPERSHELL_E2E_CLIENT_SECRET:default}` placeholders resolve to their defaults and leave Kind/local/stage realms inert (GitHub IdP disabled, e2e client secret `e2e-secret`). +6. **Developer-tier principal + impersonation (PR-ENV-9):** seed the `gateway:viewer` / `openshell-user` principal and enable admin impersonation of it for the interactive developer-boundary check. The programmatic token-exchange path (`hypershell-e2e`) is already enabled. + +**Wave plan:** W1 grant-agnostic driver auth + `hypershell-e2e` client (fully unit-testable via `tests/e2e/openshift_driver_test.sh`, no cluster) -> W2 declarative realm brokering/roles/impersonation (env-gated base realm data + optional `hypershell-github-oauth` Secret; no post-boot admin-API script) -> W3 the PR-environment workflow (identity, CD, swap, e2e, comment, trust) -> W4 timebox + reaper -> W5 pr-test deprecation + docs. W1 and W5 are the only fully-verifiable-here slices; W2-W4 need a live OpenShift cluster + GitHub OAuth App to validate end to end (W2 additionally gated on a Kind smoke test of the placeholder defaults). ### oidc-integration.spec.md @@ -1297,3 +1334,8 @@ label-selected pod informer. | 2026-09-02 | 06d6c56 | Post-connect polish: remove sample-data banner | 85% (unchanged) | Removed `usesSampleData` provider flag, inline info `Alert`, and `app.dashboard.sampleData.*` i18n keys after all OP-DASH-08 metrics connected. Operational dashboard 17/17 present. | | 2026-09-03 | 6ab016a+6583d2c | Executed OP-W2: partial metric-source failure | 85% (unchanged) | Independent adapter sources with `Promise.allSettled`; `dashboard-metric-sources.ts` stale-merge on refetch; `dashboard.metrics.partial-failure` probe; platform spec CM/CC/CLP/CLN-07 + RU-07 aligned to OP-DASH-19. Operational dashboard 19/19 present. | | 2026-09-03 | 56befbf | Reconcile: OP-DASH-18/19/20 verification | 85% (unchanged) | Verified OP-DASH-18 (NaN/Infinity fallback), OP-DASH-19 (partial failure), OP-DASH-20 (section titles + last-refreshed header + layout v23). All `operational-dashboard-ui` and adapter tests pass. Operational dashboard 20/20 present. | +| 2026-09-09 | `21f02a0` | Scoped reanalysis of e2e-testing + local-development for the new OpenShift E2E CI content | E2E Testing 100% -> 95% (1 deferred) | The HYPERSHELL-240 docs commit added `E2E_OIDC_GRANT`, the merge-queue Kind CI gate, and OpenShift-driver contract wording. Verified in code: OpenShift driver unified with Kind (#232/#244), `configure/restore_namespace_gc_timing`, `merge_group` gate in `e2e.yml` (per-component `on-merge-queue-` waits, browser-trace skip), and `.tekton/*-merge-queue.yaml` are all implemented (E2E-10/11/12 present). Only gap is D-E2E-OIDC: driver token fns hardcode `grant_type=password`; the `client_credentials`+token-exchange path is owned by `ephemeral-pr-environments.spec.md` (out of scope) and flagged as a divergence pending a scope decision. No code changed this pass. | +| 2026-09-11 | working tree | PR-ENV-8 org-gate via BFF (no custom Keycloak image) | Ephemeral PR Environments 86% -> 91% | Enforced GitHub org membership / allowlist in the web-console BFF after OIDC callback. Denied users get no HyperShell session, so the BFF never forwards an API bearer. Kind stays ungated (`GITHUB_ORG_GATE` unset). No Keycloak SPI/custom image. | +| 2026-09-09 | working tree | HYPERSHELL-240 W2 made declarative (config-as-data) | Ephemeral PR Environments 86% (unchanged) | Replaced the imperative post-boot admin-API script with env-gated realm data. `deploy/base/keycloak/keycloak.yaml` now carries the GitHub IdP (`enabled: ${PR_ENV_GITHUB_IDP_ENABLED:false}`), the two hardcoded-role `identityProviderMappers` (`platform:admin`+`gateway:creator`), the `github.org.gate`/`github.username.allowlist` realm attributes, and `hypershell-e2e` `secret: ${HYPERSHELL_E2E_CLIENT_SECRET:e2e-secret}`, all resolved at import from the optional `hypershell-github-oauth` Secret wired into the Keycloak Deployment as `optional: true` secretKeyRefs. Kind/local/stage have no Secret, so every placeholder resolves to its default and the realm is inert (IdP off, secret `e2e-secret`) - consistent with the realm already leaving unresolved `${client_id}` mappers literal. Deleted `scripts/ci/configure-github-broker.sh`; the workflow now provisions the Secret (per-PR random e2e secret, masked) into the `-keycloak` namespace *before* `openshift-up` and fails closed on missing OAuth cfg; `read-e2e-client-secret.sh` reads `hypershell-github-oauth/e2e-client-secret`. Validated: realm JSON parses, `kustomize build deploy/base/keycloak` + `deploy/openshift` render, yamllint + `make ci-test` (21/21) clean. **One gate before merge:** Kind smoke test that Keycloak resolves `${VAR:default}` (blast radius = realm import) - recorded in the handoff. | +| 2026-09-09 | working tree | HYPERSHELL-240 waves W2-W4: PR-env CI workflow, reaper, GitHub broker | Ephemeral PR Environments 14% -> 86% | **W3:** added `.github/workflows/pr-environment.yml` (origin-only `pull_request` open/reopen/synchronize/closed, per-PR concurrency, unconditional `openshift-up`, Konflux gate + digest swap, e2e with `E2E_OIDC_GRANT=client_credentials`, one marked access comment, `openshift-down` on close) + helper scripts `scripts/ci/{pr-env-lib,stamp-pr-env,swap-openshift-images-by-digest,read-e2e-client-secret,upsert-pr-comment}.sh`. **W4:** `scripts/ci/reap-pr-environments.sh` + `deploy/e2e/reaper/` CronJob/RBAC (expires-at match predicate, deletes expired `pr-*` groups only). **W2 (partial):** `scripts/ci/configure-github-broker.sh` fails closed on missing OAuth cfg, upserts the GitHub IdP (`read:org`, no RH SSO), grants `platform:admin`+`gateway:creator` on broker login, publishes the per-PR `hypershell-e2e` secret; org/allowlist ENFORCEMENT authenticator + developer-principal impersonation handed off (need Keycloak SPI + live cluster). Pure logic unit-tested: `make ci-test` 21/21 (`pr-env-lib` 19, reaper 2); `openshift_driver_test` still 20/20; yamllint + kustomize clean. Registered `scripts/ci/**`, `deploy/e2e/**`, `deploy/openshift/**`, `pr-environment.yml` under the `e2e` component. Handoff list recorded above. | +| 2026-09-09 | working tree | Began HYPERSHELL-240 (ephemeral-pr-environments) reconcile: W1 + W5 | Ephemeral PR Environments 0% -> 14% | Re-scoped to implement `ephemeral-pr-environments.spec.md` (greenfield). **W1 (PR-ENV-10 code):** made the driver token functions grant-agnostic (`E2E_OIDC_GRANT` password\|client_credentials; admin client-credentials on `hypershell-e2e`; developer Keycloak token-exchange impersonation targeting the requested audience), added `E2E_OIDC_GRANT`/`E2E_OIDC_SA_CLIENT_ID`/`E2E_OIDC_SA_CLIENT_SECRET` defaults, added the `hypershell-e2e` confidential client + service account (platform:admin+gateway:creator, `hypershell-frontend` audience mapper, standard token exchange) to the base realm, and extended `openshift_driver_test.sh` (20/20, no cluster). **W5 (PR-ENV-11):** deprecation header on `components/pr-test/e2e-openshell.sh` + DEVELOPMENT.md pointer to the shared harness; ROKS/GCP + `pr_test` CI wiring untouched. **Remaining (need live OpenShift + a GitHub OAuth App to validate):** W2 realm GitHub-brokering overlay (IdP, org gate + allowlist, dev principal, impersonation perms), W3 the PR-environment CI workflow (identity, CD lifecycle, digest swap, e2e, comment, trust boundary), W4 timebox annotation + out-of-band reaper. | diff --git a/specs/platform/e2e-testing.spec.md b/specs/platform/e2e-testing.spec.md index 396a4da1a..e3f5c8cd7 100644 --- a/specs/platform/e2e-testing.spec.md +++ b/specs/platform/e2e-testing.spec.md @@ -25,7 +25,7 @@ This spec covers the **e2e driver interface contract** (for all targets), the ** This spec owns the driver interface contract and the **OpenShift e2e driver** (`tests/e2e/drivers/openshift.sh`) so a user can run `make e2e` and `make e2e-performance` **manually** against any OpenShift cluster the user is already logged in to (via `oc login`) -- the target environment for scale and performance testing. Bring-up is a precondition: `make openshift-up` (specified in `openshift-development.spec.md`) deploys the blessed `deploy/openshift/` overlay into the current `oc` project (`OPENSHIFT_NAMESPACE` overrides), companion `${OPENSHIFT_NAMESPACE}-keycloak`, and the per-environment `${OPENSHIFT_NAMESPACE}-dev-*` cluster-scoped RBAC. This spec does not duplicate that lifecycle. Automated OpenShift pull-request CI is specified in `ephemeral-pr-environments.spec.md` (HYPERSHELL-240). The deprecation window for `components/pr-test/e2e-openshell.sh` is specified there as well. -Manual OpenShift e2e and performance runs remain in scope here. Kind CI, including the merge-queue gate, remains in scope here. The OpenShift pull-request environment job is not this spec. +Manual OpenShift e2e and performance runs remain in scope here. Kind CI, including the merge-queue gate, remains in scope here. The OpenShift pull-request *environment* (bring-up, image swap, access comment, reaping) is specified in `ephemeral-pr-environments.spec.md`. The Tests / E2E / OpenShift job that runs the suite against that environment lives in `.github/workflows/e2e.yml` and is specified here as an additional job of the CI E2E Workflow. ## Architecture @@ -334,6 +334,15 @@ The admin OIDC token from area 1 authenticates the API calls in areas 2--8 and 1 - THEN the test SHALL poll the API until the gateway phase is `Running` or `E2E_PROVISION_TIMEOUT` seconds have elapsed - AND a timeout SHALL be reported as a test failure +#### Scenario: Seeded Cluster and Release Discovery + +- GIVEN the HyperShell API is reachable and the suite has an admin bearer token +- WHEN area 2 looks up the seeded managed cluster and gateway release +- THEN it SHALL query `GET /managed_clusters` and `GET /gateway_releases` through `api_curl` and select by `E2E_SEED_CLUSTER_NAME` / `E2E_SEED_RELEASE_NAME` +- AND on `E2E_INFRA_DRIVER=kind` those names SHALL default to `local-kind` / `dev-release` +- AND on `E2E_INFRA_DRIVER=openshift` those names SHALL default to `local-openshift` / `dev-release` +- AND when either id is missing, the suite SHALL fail the area and print whether each list body was empty, an API `Error` (code and reason), or unparseable, plus a re-seed hint (`SEED_STRICT=true make openshift-seed` or `make kind-seed`) + #### Scenario: Infrastructure Verification - GIVEN a gateway has reached `Running` phase @@ -380,7 +389,7 @@ count is an advisory recent value that may lag real time (see ### Requirement: Developer RBAC Enforcement -The e2e test suite SHALL verify the RBAC boundary of the `openshell-user` tier by exercising both an operation it is allowed to perform and one it is not. The `developer` user (credentials `E2E_DEV_USERNAME` / `E2E_DEV_PASSWORD`) maps to `gateway:viewer` -> `openshell-user` per `specs/security/rbac-enforcement.spec.md`. This tier is a legitimate *user* of a gateway it can reach: it MAY create sandboxes on that gateway (the `openshell-user` role is authorized for sandbox create/list/exec per `specs/platform/openshell-gateway-oidc.spec.md`), but it is NOT a `gateway:creator`, so it MUST NOT be able to create gateways via the HyperShell API. The suite SHALL assert both halves -- the allowed operation succeeds and the denied operation returns `403 Forbidden`. +The e2e test suite SHALL verify the RBAC boundary of the `openshell-user` tier by exercising both an operation it is allowed to perform and one it is not. The `developer` user (credentials `E2E_DEV_USERNAME` / `E2E_DEV_PASSWORD`) maps to `gateway:viewer` -> `openshell-user` per `specs/security/rbac-enforcement.spec.md`. This tier is a legitimate *user* of a gateway it can reach: it MAY create sandboxes on that gateway (the `openshell-user` role is authorized for sandbox create/list/exec per `specs/platform/openshell-gateway-oidc.spec.md`), but it is NOT a `gateway:creator` in Keycloak. Whether `POST /gateways` is allowed SHALL follow the API server's `RBAC_DEFAULT_ROLES`: empty (OpenShift/production) MUST return `403 Forbidden`; unset Kind default `gateway:creator` MUST return 2xx. The suite SHALL read that env from the `hypershell-api-server` Deployment rather than branching on `E2E_INFRA_DRIVER`. The sandbox half SHALL succeed in both postures. #### Scenario: Openshell User May Create a Sandbox @@ -393,13 +402,32 @@ The e2e test suite SHALL verify the RBAC boundary of the `openshell-user` tier b #### Scenario: Openshell User May Not Create a Gateway - GIVEN a valid OIDC token has been acquired for the `developer` user +- AND the API server's `RBAC_DEFAULT_ROLES` does not include `gateway:creator` (OpenShift sets the env to empty; production isolation) - WHEN the developer calls `POST /api/hypershell/v1/gateways` with that token - THEN the API SHALL return `403 Forbidden` (the developer lacks the platform-scoped `gateway:creator` role) - AND the test SHALL record a pass for the denial +#### Scenario: Openshell User May List Gateways + +- GIVEN a valid OIDC token has been acquired for the `developer` user +- AND the API server's `RBAC_DEFAULT_ROLES` does not include `gateway:creator` +- AND the developer has no per-gateway RoleBinding +- WHEN the developer calls `GET /api/hypershell/v1/gateways` +- THEN the API SHALL return 200 with a `GatewayList` body +- AND the console SHALL NOT show "Gateways could not be loaded" + +#### Scenario: Default Creator Binding Allows Gateway Create + +- GIVEN a valid OIDC token has been acquired for the `developer` user +- AND `RBAC_DEFAULT_ROLES` is unset on the API server (Kind; the process default is `gateway:creator`) +- WHEN the developer calls `POST /api/hypershell/v1/gateways` with that token +- THEN the API SHALL return 2xx (HYPERSHELL-262 default-role bootstrap) +- AND the test SHALL delete the created gateway + #### Scenario: Unexpected Success Is a Failure - GIVEN the `developer` user attempts to create a gateway +- AND `RBAC_DEFAULT_ROLES` does not include `gateway:creator` - WHEN the API returns a 2xx status despite the missing `gateway:creator` role - THEN the test SHALL record a failure (RBAC not enforced) - AND the test SHALL delete the erroneously-created gateway to leave a clean state @@ -558,7 +586,7 @@ The system SHALL provide an independently-triggered GitHub Actions workflow at ` ### Requirement: CI Unit Test Workflow -The unit-test and e2e stages SHALL be ordered by a single orchestrator workflow at `.github/workflows/tests.yml` rather than by cross-workflow status-check polling. `tests.yml` SHALL own the `pull_request`, `push` (to `main`), `merge_group`, and `workflow_dispatch` triggers, the concurrency group, and SHALL call `unit-tests.yml` and `e2e.yml` as reusable workflows (`on: workflow_call`) wired with native `needs:` edges. `unit` SHALL depend only on `detect-changes`, and `e2e` SHALL declare `needs: [detect-changes, unit]` so it starts only after the unit-test stage concludes successfully. Because GitHub Actions skips a job by default if any needed job failed OR was skipped, `e2e` SHALL also declare `if: ${{ !cancelled() && needs.detect-changes.result == 'success' && needs.unit.result != 'failure' }}`, so a PR touching only e2e-owned paths (every job inside `unit` path-filtered away, making the `unit` caller job itself resolve to `skipped`) still runs `e2e` instead of silently skipping it. This gates only the expensive stage: the Kind-based e2e run SHALL NOT start for a SHA whose unit tests failed, and such a failure SHALL surface as a clean red `Tests CI Gate` check rather than a misleading e2e environment failure. There SHALL be no in-workflow job that polls for a preceding stage's status check. The stage workflows SHALL NOT declare their own event triggers (only `workflow_call`) so they never run as standalone duplicates. `tests.yml` SHALL NOT be gated by, and SHALL NOT gate, the separate `checks.yml` workflow (see the CI Checks Workflow requirement); the two run fully concurrently. +The unit-test and e2e stages SHALL be ordered by a single orchestrator workflow at `.github/workflows/tests.yml` rather than by cross-workflow status-check polling. `tests.yml` SHALL own the `pull_request`, `push` (to `main`), `merge_group`, and `workflow_dispatch` triggers, the concurrency group, and SHALL call `unit-tests.yml` and `e2e.yml` as reusable workflows (`on: workflow_call`) wired with native `needs:` edges. `unit` SHALL depend only on `detect-changes`, and `e2e` SHALL declare `needs: [detect-changes, unit]` so it starts only after the unit-test stage concludes successfully. Because GitHub Actions skips a job by default if any needed job failed OR was skipped, `e2e` SHALL also declare `if: ${{ !cancelled() && needs.detect-changes.result == 'success' && needs.unit.result != 'failure' }}`, so a PR touching only e2e-owned paths (every job inside `unit` path-filtered away, making the `unit` caller job itself resolve to `skipped`) still runs `e2e` instead of silently skipping it. This gates only the expensive stage: the Kind-based e2e run SHALL NOT start for a SHA whose unit tests failed, and such a failure SHALL surface as a clean red `Tests CI Gate` check rather than a misleading e2e environment failure. There SHALL be no in-workflow job that polls for a preceding Tests or Checks stage's status check. The one cross-workflow poller exception is Tests / E2E / OpenShift waiting on the independent `Deploy PR environment` check (see CI E2E Workflow). The stage workflows SHALL NOT declare their own event triggers (only `workflow_call`) so they never run as standalone duplicates. `tests.yml` SHALL NOT be gated by, and SHALL NOT gate, the separate `checks.yml` workflow (see the CI Checks Workflow requirement); the two run fully concurrently. Change detection SHALL run exactly once per workflow, in a `detect-changes` job in `tests.yml` (invoking `.github/scripts/detect-components.sh`), whose per-component outputs are passed into each stage as `with:` inputs; the stage workflows SHALL NOT detect changes internally and SHALL gate their jobs on `inputs.`. Because each stage is a reusable-workflow call, its individual jobs surface as `Unit / ` and `E2E / ` checks rather than a single per-stage check. `tests.yml` SHALL therefore provide a `tests-gate` job (`Tests CI Gate`) covering the `unit` and `e2e` stages together, which SHALL run with `if: always()`, read both stages' rolled-up `result` via `needs`, and fail unless `detect-changes` succeeded and neither stage failed or cancelled (a fully skipped stage SHALL pass the gate). Because it always runs, it is never left pending by path-filtered skips, so this is one of the two checks to mark required in branch protection (the other being `checks.yml`'s own `Checks CI Gate`). @@ -611,7 +639,9 @@ The root Makefile SHALL provide a `make unit-test-all` target that runs the same ### Requirement: CI E2E Workflow -The system SHALL provide a reusable GitHub Actions workflow at `.github/workflows/e2e.yml` (`on: workflow_call`) that runs the e2e test suite against a Kind cluster. It SHALL run as the final stage of `tests.yml`, which triggers on every pull request, on every merge-queue entry (`merge_group`), and on push to `main`. Like the unit stage, it SHALL receive the changed-component flags as `workflow_call` inputs and gate its jobs on those inputs rather than detecting changes itself; the `Tests CI Gate` job in `tests.yml` rolls its result (together with unit's) up into the required check, so it has no summary or gate job of its own. The orchestrator's `needs: [detect-changes, unit]` edge (with the `if:` override described in the CI Unit Test Workflow requirement, so a `unit` skip does not also skip `e2e`) SHALL ensure Kind is never created until the unit-test stage succeeds; the e2e workflow itself SHALL NOT contain a job that polls for that gate, or for the separate `checks.yml` workflow. The workflow SHALL still gate on Konflux image builds completing (an external build system it cannot order with `needs:`) and pull those images by digest -- it SHALL NOT rebuild component images itself. +The system SHALL provide a reusable GitHub Actions workflow at `.github/workflows/e2e.yml` (`on: workflow_call`) that runs the e2e test suite against Kind and, on origin pull requests, against the ephemeral OpenShift PR environment. It SHALL run as the final stage of `tests.yml`, which triggers on every pull request, on every merge-queue entry (`merge_group`), and on push to `main`. Like the unit stage, it SHALL receive the changed-component flags as `workflow_call` inputs and gate its jobs on those inputs rather than detecting changes itself; the `Tests CI Gate` job in `tests.yml` rolls its result (together with unit's) up into the required check, so it has no summary or gate job of its own. The orchestrator's `needs: [detect-changes, unit]` edge (with the `if:` override described in the CI Unit Test Workflow requirement, so a `unit` skip does not also skip `e2e`) SHALL ensure Kind is never created until the unit-test stage succeeds; the e2e workflow itself SHALL NOT contain a job that polls for that gate, or for the separate `checks.yml` workflow. The workflow SHALL still gate Kind jobs on Konflux image builds completing (an external build system it cannot order with `needs:`) and pull those images by digest -- it SHALL NOT rebuild component images itself. + +On origin `pull_request` events, `e2e.yml` SHALL also run a job named `OpenShift` (check: Tests / E2E / OpenShift). That job SHALL declare `needs: plan-images` and SHALL run only when `plan-images` sets `should_run=true`, matching Kind, so an e2e-irrelevant origin PR skips the OpenShift suite as well as Kind. The independent PR Environment workflow still deploys unconditionally. When the job runs, it SHALL poll the independent `Deploy PR environment` check until it succeeds, then run the OpenShift e2e suite against the live per-PR namespace as `ephemeral-pr-environments.spec.md` defines. GitHub Actions `needs:` cannot order independently-triggered workflows, so this poller is the allowed exception to the no-cross-workflow-poller rule for Unit vs Checks. Fork PRs, `merge_group`, and `push` SHALL skip that job (no per-PR environment). Push to `main` SHALL run the bring-up-test-tear-down OpenShift job from a dedicated workflow (`.github/workflows/e2e-openshift-main.yml`) that does not run on pull requests, so it does not appear as a skipped Tests check. #### Scenario: PR Triggers Workflow @@ -621,6 +651,16 @@ The system SHALL provide a reusable GitHub Actions workflow at `.github/workflow - WHEN the `e2e` stage runs - THEN it SHALL: check out the repository, use the changed-component flags passed in as inputs, create a Kind cluster via `make kind-up` with baseline images (overlapping cluster creation with the Konflux builds in progress), wait for each changed component's Konflux on-pull-request build to conclude, swap in the Konflux-built image digests via `scripts/kind/set-component-images.sh`, run `tests/e2e/e2e-openshell.sh` with `E2E_INFRA_DRIVER=kind`, and report the CI status +#### Scenario: OpenShift E2E Waits On Deploy PR Environment + +- GIVEN an origin pull request whose e2e-relevant components changed +- AND whose `Deploy PR environment` check is still running +- WHEN Tests / E2E / OpenShift starts +- THEN it SHALL have required `plan-images` with `should_run=true`, matching Kind +- AND it SHALL poll that check until it concludes `success` +- AND it SHALL then run `E2E_INFRA_DRIVER=openshift E2E_OIDC_GRANT=client_credentials bash tests/e2e/e2e-openshell.sh` against the per-PR namespace +- AND a fork PR, `merge_group` event, `push` event, or origin PR with `should_run=false` SHALL skip this job + #### Scenario: Tests Pass - GIVEN the e2e tests complete with 0 failures @@ -638,7 +678,8 @@ The system SHALL provide a reusable GitHub Actions workflow at `.github/workflow - GIVEN the PR modifies only files outside the e2e-relevant component paths (e.g., only `docs/` or `components/sdk-typescript/`) - WHEN the `e2e` workflow evaluates the change detection outputs -- THEN the e2e job SHALL be skipped +- THEN `plan-images` SHALL set `should_run=false` +- AND both the Kind and OpenShift e2e jobs SHALL be skipped - AND the workflow SHALL report `success` (to avoid blocking merges) #### Scenario: Infrastructure-Only Changes (No Source Components) @@ -852,6 +893,9 @@ deploy/ native `needs:` unit-tests.yml -- Tests unit-test stage (reusable, on: workflow_call) e2e.yml -- Tests e2e stage (reusable, on: workflow_call) + e2e-openshift-main.yml -- push-to-main OpenShift bring-up-test-tear-down + pr-environment.yml -- ephemeral PR env deploy (open/reopen/synchronize) + pr-environment-release.yml -- ephemeral PR env teardown (closed: merge or close) ``` `components/pr-test/e2e-openshell.sh` SHALL be deprecated as `ephemeral-pr-environments.spec.md` specifies. Removal is deferred until manual usage migrates; the ROKS variant is out of that deprecation. @@ -873,8 +917,8 @@ deploy/ | `E2E_OIDC_USERNAME` | `admin` | Admin OIDC user (member of `hypershell-admins` + `hypershell-users`) used for areas 1--8 and 11 | | `E2E_OIDC_PASSWORD` | `admin` | Password for the admin OIDC user (local dev only; unused when `E2E_OIDC_GRANT=client_credentials`) | | `E2E_OIDC_GRANT` | `password` | Token grant for `acquire_oidc_token` and `acquire_gateway_token_with_role`: `password` (Kind and manual OpenShift) or `client_credentials` (GitHub-brokered pull-request environments, see `ephemeral-pr-environments.spec.md`) | -| `E2E_SEED_CLUSTER_NAME` | `local-kind` on kind; unset otherwise | Pin seed discovery to this managed-cluster name. Unset means the first list item | -| `E2E_SEED_RELEASE_NAME` | `dev-release` on kind; unset otherwise | Pin seed discovery to this gateway-release name. Unset means the first list item | +| `E2E_SEED_CLUSTER_NAME` | `local-kind` on kind; `local-openshift` on openshift; unset otherwise | Pin seed discovery to this managed-cluster name. Unset means the first list item | +| `E2E_SEED_RELEASE_NAME` | `dev-release` on kind and openshift; unset otherwise | Pin seed discovery to this gateway-release name. Unset means the first list item | | `E2E_DEV_USERNAME` | `developer` | Standard OIDC user (`openshell-user` tier) used for the RBAC boundary assertions | | `E2E_DEV_PASSWORD` | `developer` | Password for the developer OIDC user (local dev only) | | `OPENSHELL_BIN` | `openshell` | Path to the openshell CLI binary | @@ -1005,7 +1049,7 @@ The system SHALL provide a `make e2e-performance` target. The target SHALL run ` The performance harness (`tests/e2e/e2e-performance.sh`) SHALL be infrastructure-agnostic. It SHALL call only the driver interface functions for infrastructure operations. It SHALL select the driver the same way the e2e suite does: auto-detected from the current KUBECONFIG context, with `E2E_INFRA_DRIVER` as an override. It SHALL exit with a non-zero status at startup if `E2E_INFRA_DRIVER` names a missing driver, and SHALL list the available drivers. It SHALL NOT contain any `kubectl`-only, `oc`-only, or `kind`-only command. -The harness SHALL obtain the seeded cluster, release, and managed database ids the same way the e2e suite does: it SHALL query the API through `api_curl` and reuse the shared seeding helpers in `tests/e2e/lib.sh`, never hardcoding ids. When `E2E_SEED_CLUSTER_NAME` / `E2E_SEED_RELEASE_NAME` are set, discovery SHALL select the matching name; when they are unset it SHALL take the first list item (the single-seed Kind/CI layout). On `E2E_INFRA_DRIVER=kind` those names SHALL default to the `make kind-up` seeds (`local-kind`, `dev-release`). Every diagnostic or resource-inspection command SHALL invoke the Kubernetes CLI through `$(get_cli_binary)`, so it resolves to `kubectl` on Kind and `oc` on OpenShift with no change to the harness. +The harness SHALL obtain the seeded cluster, release, and managed database ids the same way the e2e suite does: it SHALL query the API through `api_curl` and reuse the shared seeding helpers in `tests/e2e/lib.sh`, never hardcoding ids. When `E2E_SEED_CLUSTER_NAME` / `E2E_SEED_RELEASE_NAME` are set, discovery SHALL select the matching name; when they are unset it SHALL take the first list item (the single-seed Kind/CI layout). On `E2E_INFRA_DRIVER=kind` those names SHALL default to the `make kind-up` seeds (`local-kind`, `dev-release`). On `E2E_INFRA_DRIVER=openshift` they SHALL default to the `make openshift-seed` names (`local-openshift`, `dev-release`). When discovery cannot resolve both ids, it SHALL report whether each list body was an empty collection, an API `Error` (code and reason), or unparseable, and SHALL hint to re-run `SEED_STRICT=true make openshift-seed` (or `make kind-seed`). Every diagnostic or resource-inspection command SHALL invoke the Kubernetes CLI through `$(get_cli_binary)`, so it resolves to `kubectl` on Kind and `oc` on OpenShift with no change to the harness. The OpenShift driver is specified alongside this contract in `openshift-development.spec.md`; the performance harness uses it for OpenShift runs (see [Scope](#scope)). The harness SHALL contain no infra-specific code: it works with either driver with no change. OpenShift runs are manual and on-demand; the performance test is not wired into CI for any target (see [Design Decisions](#design-decisions)). diff --git a/specs/platform/ephemeral-pr-environments.spec.md b/specs/platform/ephemeral-pr-environments.spec.md index 305f859f1..1bafe05fb 100644 --- a/specs/platform/ephemeral-pr-environments.spec.md +++ b/specs/platform/ephemeral-pr-environments.spec.md @@ -23,13 +23,16 @@ keeps that environment in continuous deployment for the life of the pull request When a pull request opens, CI deploys the full stack into a per-PR ephemeral namespace group on a shared target OpenShift cluster, waits for Konflux to build the pull request's component images, swaps those images into the environment, -runs the OpenShift e2e suite against it, and posts a pull-request comment telling -the developer how to log in. When a later commit is pushed to the same pull -request, CI does not create a second environment: it reuses the existing one, -waits for Konflux to rebuild the changed images, swaps them in, reruns the e2e -suite, and updates the comment to say the environment now runs that commit. The -environment lives independently of any single CI run so a developer can use it as -a live debug and development target, and it is reaped after a fixed timebox so an +and posts a pull-request comment telling the developer how to log in. When +e2e-relevant paths changed, Tests / E2E / OpenShift waits for that deploy +check, then runs the OpenShift e2e suite against the live namespace (the same +`plan-images` / `should_run` gate Kind uses). When a later commit is pushed to +the same pull request, CI does not create a second environment: it reuses the +existing one, waits for Konflux to rebuild the changed images, swaps them in, +updates the comment to say the environment now runs that commit, and Tests / +E2E / OpenShift reruns the suite when `should_run` is true. The environment +lives independently of any single CI run so a developer can use it as a live +debug and development target, and it is reaped after a fixed timebox so an abandoned pull request cannot hold cluster resources. This spec owns the automated OpenShift pull-request CI workflow. The @@ -150,11 +153,12 @@ request. The workflow SHALL keep the pull request's environment continuously deployed to the pull request's current head commit for the life of the pull request. It SHALL trigger on origin-repository pull-request `opened`, `reopened`, and `synchronize` -(a new commit pushed to the pull-request branch), and it SHALL trigger on -`closed` (which covers both merge and close) to release the environment (see the -Timebox and Reaping requirement). It SHALL NOT trigger on `merge_group`. Kind -e2e, as `e2e-testing.spec.md` defines, remains the merge-queue gate; this -workflow does not share a namespace with a merge-queue SHA. +(a new commit pushed to the pull-request branch). A dedicated release workflow +SHALL trigger on `closed` (which covers both merge and close) to release the +environment (see the Timebox and Reaping requirement), so open and synchronize +runs do not list a skipped Release check. Neither workflow SHALL trigger on +`merge_group`. Kind e2e, as `e2e-testing.spec.md` defines, remains the +merge-queue gate; this workflow does not share a namespace with a merge-queue SHA. The workflow SHALL run only for pull requests targeting the origin repository. Fork pull requests SHALL NOT receive cluster credentials and SHALL NOT get an @@ -201,7 +205,8 @@ reconcile SHALL preserve any active per-namespace component swap the same way - AND it SHALL run `make openshift-up` to reconcile the environment - AND it SHALL wait for Konflux to build the new commit's images and swap them in by digest (see Image Gating and Swap) -- AND it SHALL rerun the e2e suite against the environment +- AND Tests / E2E / OpenShift SHALL rerun the e2e suite when `plan-images` + sets `should_run=true` - AND it SHALL update the access comment to reflect the new head commit (see Pull-Request Comment) @@ -241,7 +246,13 @@ SHALL overlap environment bring-up with the Konflux builds: it MAY run `make openshift-up` with baseline images while Konflux builds are still in flight, then wait for each changed component's build to conclude and swap that component's image by digest, so cluster reconcile time is hidden behind build -time. Unchanged components SHALL keep baseline registry images. The workflow +time. Bring-up SHALL set `SKIP_SEED=true` so the baseline image never receives the +seed POST (a request-contract change against that stale image would 400). After +the digest swap, the workflow SHALL run `make openshift-seed` so the seed +exercises this pull request's contract. That seed SHALL reuse existing named seed +resources on a later `synchronize` reconcile, except `dev-gateway`, which it SHALL +delete and recreate rather than reuse or duplicate, as `openshift-development.spec.md` +defines. Unchanged components SHALL keep baseline registry images. The workflow SHALL determine which components to wait for using the shared change-detection and Konflux-trigger-mirroring rules that `e2e-testing.spec.md` defines, so it never falls back to a baseline image while Konflux is building an image the pull request @@ -276,7 +287,11 @@ artifact across the stack. - THEN the workflow SHALL wait for the control plane's Konflux build for the new head commit - AND it SHALL swap the new control plane image into the environment by digest - before running e2e + before the `Deploy PR environment` check succeeds +- AND `make openshift-seed` SHALL reuse the existing `ManagedCluster`, + `GatewayRelease`, and `ManagedDatabase` seed resources +- AND it SHALL delete and recreate the existing `dev-gateway` rather than reuse + it or create a second Gateway named `dev-gateway` #### Scenario: Immutable digest is preferred over a mutable tag @@ -291,33 +306,42 @@ artifact across the stack. ### Requirement: E2E Execution Against the Environment After the environment is deployed and the pull request's images are swapped in, -the workflow SHALL run the OpenShift e2e suite against it, exactly as +Tests / E2E / OpenShift SHALL wait for the `Deploy PR environment` check to +succeed, then run the OpenShift e2e suite against it, exactly as `e2e-testing.spec.md` and `openshift-development.spec.md` define: it SHALL run `E2E_INFRA_DRIVER=openshift E2E_OIDC_GRANT=client_credentials bash tests/e2e/e2e-openshell.sh` against a KUBECONFIG context pointed at the environment, exercising the same test areas the Kind suite -exercises. The suite SHALL run on the pull request's first deployment and on every -later deployment for that pull request, so each commit is validated against a live -environment the same way the Kind e2e job validates each commit today. On failure -the workflow SHALL collect the diagnostics `e2e-testing.spec.md` defines. Whether +exercises. The suite SHALL run on the pull request's first e2e-relevant deployment and on +every later e2e-relevant deployment for that pull request, using the same +`plan-images` / `should_run` gate the Kind e2e job uses, so each e2e-relevant +commit is validated against a live environment the same way Kind validates it. +An origin PR that changes only e2e-irrelevant paths SHALL skip Tests / E2E / +OpenShift; the PR Environment deploy itself remains unconditional. On failure +the job SHALL collect the diagnostics `e2e-testing.spec.md` defines. Whether the suite passes or fails, the environment SHALL survive (see Timebox and Reaping), so a developer can inspect a failing run on the live environment. The e2e suite's authentication SHALL set `E2E_OIDC_GRANT=client_credentials` and use the non-interactive path this spec defines (see Automated E2E Authentication), because the environment's interactive login is GitHub-brokered and brokered users -have no password grant. +have no password grant. The suite lives in the Tests workflow, not inside the +PR Environment deploy job, so a deploy failure and an e2e failure surface as +distinct checks. -#### Scenario: E2E runs on every deployment +#### Scenario: E2E runs on every e2e-relevant deployment - GIVEN the environment is deployed and the pull request's images are swapped in -- WHEN the workflow reaches the test step +- AND `plan-images` set `should_run=true` (e2e-relevant paths changed) +- WHEN Tests / E2E / OpenShift sees the `Deploy PR environment` check succeed - THEN it SHALL run the OpenShift e2e suite against the environment -- AND it SHALL run the suite again on each later commit's deployment +- AND it SHALL run the suite again on each later e2e-relevant commit's deployment +- AND an origin PR with `should_run=false` SHALL skip this job while the + environment remains deployed #### Scenario: Environment survives a failing run - GIVEN the e2e suite fails -- WHEN the workflow finishes +- WHEN Tests / E2E / OpenShift finishes - THEN it SHALL collect the failure diagnostics - AND the environment SHALL remain deployed for developer inspection @@ -354,12 +378,28 @@ The reaper SHALL NOT delete namespaces that fail that match, including local environment identifier is not `pr-*`. It SHALL refuse reserved names (`default`, `kube-*`, `openshift-*`). -On pull-request `closed` (merge or close), the workflow SHALL release the +Gateway and ManagedDatabase namespaces are not in the namespace group and do not +carry `hypershell.redhat.io/owned`. Periodic GC cannot reap them after the +platform project is gone (`openshell-gateway-namespace-gc.spec.md`). When the +reaper deletes a pull-request platform namespace, it SHALL also delete namespaces +labeled `hypershell.redhat.io/managed=true`, +`app.kubernetes.io/managed-by=hypershell-control-plane`, and +`hypershell.redhat.io/instance=`, matching +`make openshift-down`. It SHALL also reap those instance-labeled namespaces when +the platform project is already absent and the instance identity is a +`hypershell-ci-pr-` platform name, so a previous incomplete teardown +cannot leave `openshell-*` workloads behind. It SHALL NOT delete namespaces +labeled for a different instance, including `hyp4`, `hyp5`, and local +`make openshift-up` environments. + +On pull-request `closed` (merge or close), CI SHALL release the environment as the primary path by removing the namespace group the same way -`make openshift-down` does. The timebox SHALL remain the backstop for the case -where the close event does not fire or its release cannot be confirmed; when the -release step cannot confirm the release, the workflow SHALL report the failure so -an operator can free the environment. +`make openshift-down` does. That release SHALL live in a `closed`-only workflow +so open and synchronize runs do not list a skipped Release check. The timebox +SHALL remain the backstop for the case where the close event does not fire or +its release cannot be confirmed; when the release step cannot confirm the +release, the workflow SHALL report the failure so an operator can free the +environment. #### Scenario: Deploying run refreshes the expiry @@ -376,8 +416,21 @@ an operator can free the environment. - AND the pull request was neither merged nor closed - WHEN the out-of-band reaper evaluates environments - THEN it SHALL delete the expired namespace group +- AND it SHALL delete namespaces labeled + `hypershell.redhat.io/instance=` - AND it SHALL delete only namespaces matching the pull-request ownership labels - and `pr-*` environment identifier + and `pr-*` environment identifier, plus that instance's managed gateway and + database namespaces + +#### Scenario: Leftover instance namespaces are reaped after the project is gone + +- GIVEN the platform project `hypershell-ci-pr-267` is already absent +- AND gateway namespaces remain labeled + `hypershell.redhat.io/instance=hypershell-ci-pr-267` +- WHEN the out-of-band reaper evaluates environments +- THEN it SHALL delete those leftover instance-managed namespaces +- AND it SHALL NOT delete namespaces labeled for `hyp4`, `hyp5`, or a local + `make openshift-up` environment #### Scenario: Local environments are not reaped @@ -417,37 +470,67 @@ namespace, the API Route URL, and the web-console Route URL -- presented as the same login guidance `make openshift-up` prints at the end of a successful bring-up, so the comment and the command agree. -On the pull request's first deployment, the workflow SHALL post the initial -comment once the environment is ready. On each later deployment for the same pull -request, the workflow SHALL update the marked comment to state that the -environment has been updated to commit `` and SHALL refresh the same login -details. The `` in the comment SHALL be the commit whose digest swap -completed, so the comment never claims a commit the swap did not deploy. - -The comment SHALL NOT contain any credential. It MAY include an `oc login` -template with the credential redacted (for example -`oc login --server= --token=`). The credential itself SHALL be -delivered only through a channel that only an authorized developer can read, and -SHALL be short-lived and namespace-scoped, as `openshift-development.spec.md` -requires. No kubeconfig, token, or password SHALL appear in the comment, the job -logs, or a public artifact. +The workflow SHALL post the marked comment as the first step of a deploy run, +before cluster login, deploy, or e2e. When the pull request has no marked +comment yet (first deploy), that comment SHALL state that the environment is +deploying to commit `` and SHALL contain no access facts yet. This keeps +the access comment near the top of the pull request's timeline: because it is +normally the first comment the workflow ever adds, later edits do not need to +reorder it among other bots' checks and comments. Once the environment is +ready, the workflow SHALL edit that same marked comment in place with the +access facts rather than posting a second comment. On each later deployment +for the same pull request, the workflow SHALL repeat this sequence against the +one marked comment: an early edit stating the environment is updating to the +new commit and may not be fully responsive during the update, while retaining +the existing access-fact table (namespaces, console URL, API Route URL, +web-console Route URL, and CLI login do not change from reconcile to +reconcile), then a final edit stating it has been updated to +commit `` with refreshed login details. The `` in the final comment +SHALL be the commit whose digest swap completed, so the comment never claims a +commit the swap did not deploy. + +The comment SHALL NOT contain any credential. It SHALL include an `oc login` +template using the `--web` flag (for example `oc login --server= +--web`), so OpenShift drives the developer's browser through the same +GitHub-organization-gated OAuth flow the web console uses and handles token +issuance and refresh itself. No separate credential delivery step is needed: no +kubeconfig, token, or password SHALL appear in the comment, the job logs, or a +public artifact. + +#### Scenario: Deploying placeholder posted first + +- GIVEN a pull request is opened +- WHEN the deploy job starts, before cluster login or deploy +- THEN it SHALL post one pull-request comment stating the environment is + deploying to the head commit +- AND the comment SHALL contain the hidden marker `` +- AND the comment SHALL contain no access facts or credential #### Scenario: Initial comment on pull-request open - GIVEN a pull request is opened and its environment becomes ready - WHEN the workflow finishes deploying -- THEN it SHALL post one pull-request comment with the namespaces, console URL, - API Route URL, and web-console Route URL -- AND the comment SHALL contain the hidden marker `` +- THEN it SHALL edit the marked comment in place with the namespaces, console + URL, API Route URL, and web-console Route URL, rather than posting a second + comment - AND the comment SHALL present the same login details `make openshift-up` prints - AND the comment SHALL NOT contain a credential #### Scenario: Comment updated on each new commit - GIVEN a pull request already has an access comment that carries the marker -- WHEN a new commit's digest swap completes -- THEN the workflow SHALL update that marked comment to say the environment was - updated to commit `` + and an access-fact table +- WHEN a new commit's deploy run starts +- THEN the workflow SHALL edit that marked comment to say the environment is + updating to the new commit +- AND the comment SHALL note that the environment may not be fully responsive + during the update +- AND the comment SHALL retain the existing access-fact table +- AND the workflow SHALL NOT replace the comment with the first-deploy + placeholder that has no access facts +- AND WHEN that commit's digest swap completes +- THEN the workflow SHALL edit the same marked comment again to say the + environment was updated to commit `` - AND `` SHALL be the commit whose digest swap completed - AND the workflow SHALL NOT post a second access comment - AND the comment SHALL refresh the login details @@ -457,7 +540,8 @@ logs, or a public artifact. - GIVEN the workflow delivers access details - WHEN a reader inspects the comment, the job logs, and public artifacts - THEN no kubeconfig, token, or password appears in any of them -- AND the credential is available only through a secure channel +- AND the `oc login` template uses `--web` so OpenShift issues the credential + interactively through the developer's own browser session ### Requirement: Pull-Request Trust Boundary @@ -511,36 +595,46 @@ unset, the workflow SHALL fail before the access comment is posted, rather than leave an environment nobody can log into. The Keycloak SHALL configure a GitHub identity provider using that OAuth App -and the OAuth `read:org` scope so it can read the authenticating user's -organization membership. The realm SHALL restrict which GitHub identities may -complete authentication: +and the OAuth `read:org` scope so HyperShell can read the authenticating user's +organization membership. Interactive login SHALL restrict which GitHub +identities may use the environment: - **Organization membership is the default gate.** A GitHub user who is a member of the `openshift-online` organization SHALL be allowed to authenticate. -- **An allowlist admits extra usernames outside the organization.** The realm - SHALL support an allowlist of individual GitHub usernames that MAY authenticate - even when they are not members of `openshift-online`, so an outside contributor - can log in without being added to the organization. The allowlist is - additive: it widens login beyond the organization gate, never narrows it, and - it does not grant the listed user a CI deploy. -- **Everyone else is denied.** A GitHub user who is neither an `openshift-online` - member nor on the allowlist SHALL be denied at authentication; the environment - SHALL NOT create a HyperShell session for them. - -The organization gate and the allowlist SHALL be enforced during authentication -(for example through a first-broker-login flow step or an equivalent authenticator -that checks `read:org` membership and the configured allowlist), not merely by -post-hoc role assignment, so a denied user never obtains a token. The organization -name, the allowlist, the GitHub OAuth client id and secret, and the stable -callback URL SHALL come from configuration, not code, so a different +- **An allowlist admits extra usernames outside the organization.** The + environment SHALL support an allowlist of individual GitHub usernames that MAY + authenticate even when they are not members of `openshift-online`, so an + outside contributor can log in without being added to the organization. The + allowlist is additive: it widens login beyond the organization gate, never + narrows it, and it does not grant the listed user a CI deploy. +- **Everyone else is denied.** A GitHub user who is neither an + `openshift-online` member nor on the allowlist SHALL be denied a HyperShell + session. + +The organization gate and the allowlist SHALL be enforced by the web-console BFF +after the OIDC callback, using the Keycloak-stored GitHub token +(`storeToken`) to call GitHub `GET /user/orgs` and comparing +`preferred_username` against the allowlist. This is a weaker guarantee than a +Keycloak first-broker-login SPI: Keycloak may still issue an SSO session, but +the BFF SHALL NOT persist a HyperShell session for a denied user. The console's +API bearer is that session's access token, so a denied login SHALL NOT produce a +token the BFF can forward. The API server is not separately org-gated; e2e and +control-plane callers keep using their own service-account clients. These are +developer environments; the BFF check avoids a custom Keycloak image. Kind and +local SHALL leave `GITHUB_ORG_GATE` unset when the GitHub Secret is absent. When +the gate is on, a Keycloak session with no readable GitHub broker identity +(seeded password users: Keycloak `GET /broker/github/token` returns 403 or 404) +SHALL still receive a HyperShell session. GitHub-brokered identities remain +subject to the organization and allowlist checks. +The organization name, the allowlist, the GitHub OAuth client id and secret, and +the stable callback URL SHALL come from configuration, not code, so a different organization, allowlist, or OAuth App does not require an overlay edit. #### Scenario: Organization member authenticates - GIVEN a GitHub user who is a member of `openshift-online` - WHEN they log in to a pull-request environment through GitHub -- THEN Keycloak SHALL allow the authentication -- AND SHALL create their HyperShell session +- THEN the web console SHALL create their HyperShell session #### Scenario: Allowlisted non-member authenticates @@ -548,15 +642,17 @@ organization, allowlist, or OAuth App does not require an overlay edit. - AND that username is on the environment's allowlist - AND an origin-repo pull request has already deployed the environment - WHEN they log in through GitHub -- THEN Keycloak SHALL allow the authentication +- THEN the web console SHALL create their HyperShell session - AND that allowlist entry SHALL NOT have caused CI to deploy a fork pull request #### Scenario: Non-member, non-allowlisted user is denied - GIVEN a GitHub user who is neither an `openshift-online` member nor allowlisted - WHEN they attempt to log in through GitHub -- THEN Keycloak SHALL deny the authentication -- AND SHALL NOT issue a token or create a session +- THEN the web console SHALL deny the login +- AND SHALL NOT create a HyperShell session +- AND SHALL show an access-denied error in the console +- AND SHALL NOT forward an API bearer on later `/api/*` calls from that login #### Scenario: GitHub redirects to the stable callback @@ -568,6 +664,24 @@ organization, allowlist, or OAuth App does not require an overlay edit. - AND the callback SHALL complete the broker login against that pull request's Keycloak +#### Scenario: Keycloak recycle keeps the console redirect URIs + +`start-dev --import-realm` only loads the rendered realm into an empty data +dir. Recycle after an OAuth-secret change SHALL re-import +`hypershell-frontend` redirect URIs for the web-console Route, not the +localhost defaults from the overlay. `make openshift-up` SHALL stamp +`HYPERSHELL_CONSOLE_HOST` on the `render-realm-config` init container so that +import is durable. The workflow SHALL skip the recycle when the oauth-secret +annotation already matches. + +- GIVEN `make openshift-up` has stamped the web-console Route host on + `render-realm-config` +- WHEN CI recycles Keycloak because the GitHub OAuth secret hash changed +- THEN `--import-realm` re-imports `hypershell-frontend` redirect URIs for that + console host +- AND Keycloak SHALL NOT reject the BFF `redirect_uri` with + `Invalid parameter: redirect_uri` + #### Scenario: Missing GitHub OAuth configuration fails bring-up - GIVEN the GitHub OAuth client id, client secret, or stable callback URL is unset @@ -640,9 +754,18 @@ the service-account and token-exchange path so the suite still calls those functions. The realm SHALL include a dedicated confidential client `hypershell-e2e` whose -service account holds `platform:admin` and `gateway:creator`. The workflow SHALL -NOT reuse `hypershell-provisioner` for e2e (that client holds `manage-clients` -and `manage-users`). After `make openshift-up`, CI SHALL read the `hypershell-e2e` +service account holds `platform:admin` and `gateway:creator`. That client, its +service-account user, and the `realm-management: impersonation` +clientScopeMapping SHALL be present in the **imported** realm only when +`HYPERSHELL_E2E_CLIENT_ENABLED` is true (the `hypershell-github-oauth` Secret +in a pull-request environment). Kind, local OpenShift without that Secret, and +hub/ibm SHALL import a realm that omits that client, user, and mapping -- not a +disabled copy. A disabled leftover from an earlier import SHALL still be a +no-op: the control plane SHALL grant FGAP v1 token-exchange onto gateway and +frontend clients only when `hypershell-e2e` exists **and is enabled**. The +workflow SHALL NOT reuse `hypershell-provisioner` for e2e (that client holds +`manage-clients` and `manage-users`). After `make openshift-up`, CI SHALL read +the `hypershell-e2e` client secret from the deployed Keycloak namespace (a Kubernetes Secret in `hypershell-ci-pr--keycloak`) and SHALL NOT take it from a repo secret that cannot match a per-PR realm. The e2e suite's admin `acquire_oidc_token` @@ -671,6 +794,22 @@ appear in logs, the pull-request comment, or public artifacts. - AND CI SHALL have read that client secret from the Keycloak namespace after `make openshift-up` +#### Scenario: Kind and hub omit the e2e identity + +- GIVEN Kind, local OpenShift without `hypershell-github-oauth`, or hub/ibm +- WHEN Keycloak imports the rendered realm +- THEN the imported realm SHALL NOT contain client `hypershell-e2e` +- AND SHALL NOT contain user `service-account-hypershell-e2e` +- AND SHALL NOT contain `clientScopeMappings` for `hypershell-e2e` + +#### Scenario: Present-but-disabled e2e client does not mutate production realms + +- GIVEN a Keycloak realm that still has client `hypershell-e2e` with `enabled: false` +- WHEN the control plane reconciles a gateway client +- THEN `EnsureE2ETokenExchange` SHALL skip +- AND it SHALL NOT enable `admin-fine-grained-authz` on `realm-management` +- AND it SHALL NOT attach a token-exchange policy to the gateway or frontend client + #### Scenario: CI acquires the developer HyperShell API token by impersonation - GIVEN the seeded developer-tier principal exists in the realm @@ -788,6 +927,7 @@ exists). | Namespace name from the pull-request number (`hypershell-ci-pr-`) | A short, stable, collision-free identifier that every run for a pull request derives without external state; fits well within the DNS-label bound that keeps `-keycloak` under 63 characters. Branch names and commit SHAs are not stable for the life of one pull request | | Same lifecycle labels as `make openshift-up`, with `pr-` as the environment id | Reuses `hypershell.redhat.io/owned` and `hypershell.redhat.io/environment` so status and cleanup tooling stay one selector set; the `pr-` prefix lets the reaper ignore local environments. CI must be able to patch namespaces; failing closed beats an unlabeled environment the reaper cannot see | | `make openshift-up` on every deploying trigger, unconditionally | The command is already idempotent and reconciling, so one code path creates on first run and reconciles on later runs; branching on "does it exist" would duplicate logic and risk drift | +| Seed after every image swap; reuse existing named resources, except `dev-gateway` | `SKIP_SEED` on `openshift-up` keeps the baseline image from seeing the seed POST; `make openshift-seed` after the swap exercises this PR's contract. Gateway names are not unique, so later reconciles must look up `dev-gateway` (and the other seed names) rather than POST a second copy. `dev-gateway` is the one exception: Keycloak runs on in-memory storage with no persistent volume, so a Keycloak pod restart discards its dynamically-provisioned OIDC client while the `dev-gateway` row survives untouched in PostgreSQL, and the reconciler deliberately never auto-recreates a missing client (`openshell-gateway-keycloak.spec.md`, "Existing gateway client is missing"). Reusing a `dev-gateway` that predates the current Keycloak instance would permanently strand it in status `Keycloak client is missing`, so seeding deletes and recreates it on every run instead. This is a stopgap until Keycloak has durable storage across restarts | | CI stamps `hypershell.redhat.io/expires-at`; `make openshift-up` does not | The timebox is a pull-request cost bound, not a local-dev contract. Stamping from the workflow after bring-up refreshes active PRs without time-boxing developer namespaces | | Origin `pull_request` only; Kind remains the merge-queue gate | `merge_group` has no stable pull-request number the way this namespace is keyed, and would race a `synchronize` swap on the same namespace. Fork PRs must not receive cluster credentials; the allowlist is login, not deploy | | Per-PR concurrency group | Two in-flight swaps on one namespace can leave mixed digests; cancelling or queuing the older run keeps the comment SHA honest | @@ -795,9 +935,10 @@ exists). | Hidden HTML comment marker | Later runs have to find "the" access comment; a stable marker avoids editing an unrelated comment or posting duplicates | | Immutable digests over untrusted tags | The environment runs exactly the artifact CI verified; pinning by `@sha256:` means a tag that is later re-pushed cannot silently change what the environment runs. A tag is a last-resort fallback only when no digest exists, and the fallback is recorded rather than silent | | Close releases as primary path, timebox as backstop | The merge/close event frees the environment promptly in the common case; the timebox covers the case where the event does not fire or release cannot be confirmed | +| Reap instance-labeled gateway namespaces with the namespace group | Gateway and ManagedDatabase namespaces are siblings of the platform project, not inside it. Periodic GC dies with the controller, so down and the reaper must delete `hypershell.redhat.io/instance=` or e2e leftovers stay on the shared cluster | | One updated comment per pull request, carrying the completed-swap commit SHA | The pull request shows the live environment's current state instead of a growing list of stale comments; pinning the SHA whose digest swap completed prevents claiming a commit the swap did not deploy | | GitHub brokering, not Red Hat SSO | These are developer/debug environments; GitHub identity plus an organization gate and allowlist lets an outside contributor log in to an origin-repo environment, where Red Hat SSO would tie the environment to production identity | -| Organization gate by default, allowlist for extras | Organization membership is the common case; the additive allowlist admits outside contributors to login without adding them to the organization. Enforcing both during authentication (not by post-hoc roles) means a denied user never gets a token | +| Organization gate by default, allowlist for extras | Organization membership is the common case; the additive allowlist admits outside contributors to login without adding them to the organization. Enforcing both at BFF login is sufficient: the console API bearer only exists after a HyperShell session is created, so a denied user never receives one. A custom Keycloak image is not required | | Authenticated users get `platform:admin` and `gateway:creator`; developer tier by impersonation | `platform:admin` is view and delete only; create requires `gateway:creator`. A single GitHub identity federates to one Keycloak user, so there is no admin-or-developer account picker. A seeded `gateway:viewer` / `openshell-user` principal plus impersonation lets an admin still verify the developer boundary with the same login | -| Dedicated `hypershell-e2e` client; secret read from the deployed Keycloak | Brokered GitHub users have no password grant. A per-PR realm cannot share a repo-held provisioner secret, and `hypershell-provisioner` is too privileged (`manage-clients` / `manage-users`). Token exchange onto the HyperShell API client and onto the per-gateway client covers area 9 without a password grant. `E2E_OIDC_GRANT` keeps Kind and manual OpenShift on the password grant | +| Dedicated `hypershell-e2e` client, imported only when enabled | Brokered GitHub users have no password grant. A per-PR realm cannot share a repo-held provisioner secret, and `hypershell-provisioner` is too privileged (`manage-clients` / `manage-users`). Token exchange onto the HyperShell API client and onto the per-gateway client covers area 9 without a password grant. Omitting the client from Kind/local/hub imports (and gating control-plane grants on `enabled==true`) keeps the impersonation identity out of production reconcile paths. `E2E_OIDC_GRANT` keeps Kind and manual OpenShift on the password grant | | Deprecate `e2e-openshell.sh` now, remove it later; leave ROKS alone | This workflow is the canonical pull-request OpenShift e2e path, so the legacy `e2e-openshell.sh` is superseded. Team members still run it, so it is deprecated first (notice + docs pointing at the shared harness) and removed later once that usage migrates. New coverage lands only in `tests/e2e/`. The ROKS variant is out of scope; the `pr_test` component stays until both scripts are gone | diff --git a/specs/platform/local-development.spec.md b/specs/platform/local-development.spec.md index ead9f6c79..9e9a3cd9e 100644 --- a/specs/platform/local-development.spec.md +++ b/specs/platform/local-development.spec.md @@ -99,7 +99,7 @@ Keycloak SHALL be deployed into the Kind cluster by default. When the `KIND_KEYC ### Gateway Resource -`make kind-up` SHALL seed all resources needed for a functional local environment: ManagedCluster, GatewayRelease, ManagedDatabase (`openshell-db` when `DATABASE_PROVIDER=cnpg`), and a Gateway with OIDC configuration pointing at the local Keycloak instance. The ManagedDatabase seed triggers the ManagedDatabaseReconciler to create the gateway database CNPG Cluster infrastructure. When a single ManagedDatabase exists, gateways created without an explicit `database_id` are auto-assigned to it. The seeding step obtains a Bearer token from Keycloak using the admin user (not the control-plane service account), then creates each resource via the REST API. If any seed step fails (e.g. the resource already exists from a previous run), it SHALL warn and continue rather than abort. This makes `kind-up` fully self-contained -- a developer gets a working gateway without any manual API calls after the initial setup. +`make kind-up` SHALL seed all resources needed for a functional local environment: ManagedCluster, GatewayRelease, ManagedDatabase (`openshell-db` when `DATABASE_PROVIDER=cnpg`), and a Gateway with OIDC configuration pointing at the local Keycloak instance. The ManagedDatabase seed triggers the ManagedDatabaseReconciler to create the gateway database CNPG Cluster infrastructure. When a single ManagedDatabase exists, gateways created without an explicit `database_id` are auto-assigned to it. The seeding step obtains a Bearer token from Keycloak using the admin user (not the control-plane service account), then creates each resource via the REST API. Seeding SHALL be reuse-or-create for the named seed resources (`local-kind`, `dev-release`, `openshell-db`, `dev-gateway`): when a resource with that name already exists, the command SHALL reuse its id and SHALL NOT create another. If any seed step fails, it SHALL warn and continue rather than abort, unless `SEED_STRICT=true`. This makes `kind-up` fully self-contained -- a developer gets a working gateway without any manual API calls after the initial setup. The local environment SHALL NOT deploy the gateway's PostgreSQL directly - the control plane reconciler provisions a dedicated database and role for each gateway in the shared CNPG Cluster using CNPG `Database` and `DatabaseRole` CRDs (see `specs/platform/openshell-gateway-database.spec.md`). This ensures the local environment exercises the same database provisioning path used in production. The API server's database is also managed by CNPG via a separate Cluster CR in `hypershell-system` (see Cluster-Level Prerequisites above). diff --git a/specs/platform/openshell-gateway-namespace-gc.spec.md b/specs/platform/openshell-gateway-namespace-gc.spec.md index 950de8c28..7e0053961 100644 --- a/specs/platform/openshell-gateway-namespace-gc.spec.md +++ b/specs/platform/openshell-gateway-namespace-gc.spec.md @@ -175,6 +175,13 @@ Reaping SHALL be best-effort and idempotent, and SHALL only ever delete gateway workload namespaces owned by this instance (matching the gateway prefix, not the database prefix, and carrying this instance's identity label). +Environment teardown is a separate path. When the platform project is deleted, +this controller is gone and cannot run periodic GC. `make openshift-down` and the +pull-request reaper SHALL delete namespaces labeled +`hypershell.redhat.io/instance=` as +`openshift-development.spec.md` and `ephemeral-pr-environments.spec.md` define, +including ManagedDatabase namespaces that this sweep excludes. + #### Scenario: Orphaned gateway namespace reaped after grace period - GIVEN a namespace owned by this control-plane instance with a gateway-prefixed diff --git a/specs/platform/openshift-development.spec.md b/specs/platform/openshift-development.spec.md index ecbb9dffd..5f28f0a35 100644 --- a/specs/platform/openshift-development.spec.md +++ b/specs/platform/openshift-development.spec.md @@ -197,7 +197,24 @@ Like `make kind-up`, `make openshift-up` SHALL seed the domain resources a developer needs for a working gateway -- a ManagedCluster, a GatewayRelease, a ManagedDatabase, and a Gateway -- with the OpenShift Route and OIDC values for the environment, so that one command produces a working gateway and -the OpenShift workflow matches the Kind workflow. +the OpenShift workflow matches the Kind workflow. Seeding SHALL be reuse-or-create +for the named seed resources `local-openshift`, `dev-release`, and `openshell-db`: +when a resource with that name already exists, the command SHALL reuse its id and +SHALL NOT POST a second copy. Gateway names are not unique in the API, so a second +`make openshift-up` or `make openshift-seed` against a namespace that already has a +`dev-gateway` SHALL leave a single Gateway with that name -- but for `dev-gateway` +specifically, "leave a single Gateway with that name" SHALL mean deleting the +existing one and creating a fresh one, not reusing it. Keycloak runs on in-memory +storage with no persistent volume (see the OpenShift Development Environment +Overlay's Keycloak Deployment), so a Keycloak pod restart discards `dev-gateway`'s +dynamically-provisioned OIDC client while its row survives untouched in +PostgreSQL; reusing that stale `dev-gateway` would permanently strand it in status +`Keycloak client is missing`, since the GatewayReconciler deliberately never +auto-recreates a missing client (`openshell-gateway-keycloak.spec.md`, "Existing +gateway client is missing"). This is a stopgap until Keycloak has durable storage +across restarts. This keeps seed safe to re-run on every reconcile of a +long-lived environment (ephemeral pull-request environments re-run +`make openshift-seed` after each image swap). The platform's own database provider (CNPG `Cluster` vs. the bundled PostgreSQL Deployment) SHALL be selectable with `DATABASE_PROVIDER=cnpg|deployment`, mirroring @@ -241,7 +258,26 @@ HyperShell environment. The command SHALL wait until each project is gone before rather than return after it has only requested deletion. When `oc delete project` is forbidden, the command SHALL delete HyperShell resources inside both projects (including the bundled Keycloak workload, which is unlabeled), wait for those -deletes, and leave the projects. The +deletes, and leave the projects. + +Gateway and ManagedDatabase workloads do not live in the platform project. The +control plane creates sibling namespaces (`openshell-`, `openshell-db-`) +stamped with `hypershell.redhat.io/instance=` as +`openshell-gateway-namespace-gc.spec.md` defines. Periodic GC cannot reap those +after the platform project is gone, because only that instance's controller +selects on its own identity, and deleting the project kills the controller. +`make openshift-down` SHALL therefore delete every namespace labeled +`hypershell.redhat.io/managed=true`, +`app.kubernetes.io/managed-by=hypershell-control-plane`, and +`hypershell.redhat.io/instance=`, including ManagedDatabase +namespaces, after the platform project is removed (or when that project is already +absent) so the controller cannot recreate them from API state. It SHALL NOT +delete namespaces labeled for a different instance. An empty instance identity +SHALL refuse that selector rather than match unlabeled leftovers. This cleanup +SHALL still run when the platform project is already gone, so a previous partial +down can be completed with the same command. + +The `make openshift-status` command SHALL report the cluster, the environment namespaces, the pods, the services, the Routes, the Gateway status, and the component swap state, the same categories that `make kind-status` reports. @@ -293,26 +329,66 @@ its hostname rather than `oc exec`. The imported realm only allows `hypershell-frontend` redirect URIs to the web-console Route origin (`https:///auth/callback` and `https://`) so the BFF authorization-code callback succeeds. Wildcard redirect URIs SHALL -NOT be registered. +NOT be registered. The console host SHALL be stamped on the +`render-realm-config` init container as `HYPERSHELL_CONSOLE_HOST`, not only on +the `keycloak` container: `oc set env` without `-c` only patches +`spec.containers` on OpenShift, and `start-dev --import-realm` on an empty H2 +store (a pod recycle) would otherwise restore the localhost defaults and +Keycloak would reject the BFF callback (`Invalid parameter: redirect_uri`). +The stamp SHALL be a strategic-merge patch of those two env vars. A +get-modify-replace of the live Deployment races status updates and fails with +`the object has been modified`. - GIVEN a developer runs `make openshift-up` - WHEN the deployment is ready - THEN `hypershell-frontend` redirect URIs include the web-console Route `/auth/callback` +- AND the `render-realm-config` init container env `HYPERSHELL_CONSOLE_HOST` is + the web-console Route host - AND the driver obtained the seed API token from the Keycloak Route - AND Keycloak accepts the BFF `redirect_uri` for that console host +- AND Keycloak still accepts that `redirect_uri` after a pod recycle that + re-runs `--import-realm` + +#### Scenario: Seeding reuses existing named resources + +- GIVEN the environment already has a ManagedCluster named `local-openshift`, a + GatewayRelease named `dev-release`, and a ManagedDatabase named `openshell-db` +- WHEN the developer runs `make openshift-up` or `make openshift-seed` +- THEN the command reuses those existing resources + +#### Scenario: Seeding always recreates dev-gateway + +- GIVEN the environment already has a Gateway named `dev-gateway` +- WHEN the developer runs `make openshift-up` or `make openshift-seed` +- THEN the command deletes the existing `dev-gateway` +- AND it creates a new Gateway named `dev-gateway` +- AND it does not leave two Gateways named `dev-gateway` #### Scenario: Remove the deployment - GIVEN a HyperShell deployment exists from `make openshift-up` +- AND that environment has created gateway and ManagedDatabase namespaces labeled + `hypershell.redhat.io/instance=` - WHEN the developer runs `make openshift-down` or `make openshift-teardown` - THEN the scripts delete the platform project and the companion `-keycloak` project - AND the command does not return until both projects are gone, or until project deletion is forbidden and HyperShell resources in both projects have been removed - AND when project deletion is forbidden, the scripts remove HyperShell resources from both projects, including Keycloak +- AND the scripts delete namespaces labeled for this instance, including + `openshell-*` and `openshell-db-*` +- AND the scripts do not delete namespaces labeled for a different instance - AND the scripts do not delete resources that belong to other environments or to cluster infrastructure +#### Scenario: Down reaps leftover instance namespaces after the project is gone + +- GIVEN the platform project `hypershell-ci-pr-267` is already absent +- AND gateway namespaces remain labeled `hypershell.redhat.io/instance=hypershell-ci-pr-267` +- WHEN the developer runs `OPENSHIFT_NAMESPACE=hypershell-ci-pr-267 make openshift-down` +- THEN the scripts delete those leftover instance-managed namespaces +- AND the scripts do not delete namespaces labeled `hyp4` or `hyp5` + #### Scenario: No target cluster is available - GIVEN the developer has no reachable OpenShift cluster in the current kubeconfig @@ -400,7 +476,8 @@ the projects. The command SHALL NOT require namespace labels in order to delete. - GIVEN two HyperShell environment namespace groups exist on one cluster - WHEN a developer runs `make openshift-down` for one environment - THEN the scripts remove only that environment's platform project and `-keycloak` project, or the HyperShell resources in them -- AND the other environment stays intact +- AND the scripts delete that environment's instance-labeled gateway and database namespaces +- AND the other environment stays intact, including its instance-labeled namespaces #### Scenario: Deployment refuses a foreign namespace @@ -570,11 +647,16 @@ The swap build SHALL target the OpenShift node architecture, not the laptop architecture. When `SWAP_PLATFORM` is set (`linux/amd64` or `linux/arm64`), the driver SHALL use that architecture. When it is unset, the driver SHALL read the architecture from the cluster nodes. The driver SHALL pass -`--platform linux/` to the container build. Component Dockerfiles SHALL -pin Red Hat Hardened Image manifests per architecture (`amd64` and `arm64`) -and SHALL select the pin with `TARGETARCH` (and `BUILDARCH` for a native Go -toolchain). A single-arch pin SHALL NOT be used: that produces `Exec format -error` when an arm64 laptop image is pulled by amd64 nodes. +`--platform linux/` to the container build and SHALL pass `TARGETARCH` +for that architecture. Component Dockerfiles SHALL pin Red Hat Hardened Image +manifests per architecture (`amd64` and `arm64`) and SHALL select the runtime +pin with `TARGETARCH`. Compile stages that cannot run under qemu SHALL select +a native toolchain with `BUILDARCH` from the laptop architecture (`uname -m`): +Go cross-compiles with `GOARCH=${TARGETARCH}`; the web-console Vite/esbuild +step SHALL run on the `BUILDARCH` Node image, then install production native +addons (including `sodium-native`) for `TARGETARCH`. A single-arch pin SHALL +NOT be used: that produces `Exec format error` when an arm64 laptop image is +pulled by amd64 nodes. Because more than one developer can share one cluster, each working-tree image SHALL have an immutable identity scoped to the source commit and to @@ -621,9 +703,21 @@ build, which run the baseline image, and the exact image each one runs. - AND the OpenShift nodes are amd64 - WHEN the developer runs `make openshift-api-server-up` - THEN the scripts build the API server with `--platform linux/amd64` -- AND the Dockerfiles select the amd64 HI digest pins +- AND the scripts pass `BUILDARCH=arm64` and `TARGETARCH=amd64` +- AND the Dockerfiles select the amd64 HI digest pins for the runtime - AND the migrate init container SHALL start without `Exec format error` +#### Scenario: Swap web console compiles Vite natively + +- GIVEN the developer laptop is arm64 +- AND the OpenShift nodes are amd64 +- WHEN the developer runs `make openshift-web-console-up` +- THEN the scripts pass `BUILDARCH=arm64` and `TARGETARCH=amd64` +- AND `react-router build` (Vite/esbuild) SHALL run on the arm64 Node builder, + not under qemu for linux/amd64 +- AND the runtime image SHALL be linux/amd64 with production native addons + installed for amd64 + #### Scenario: Swap without SWAP_REGISTRY stops - GIVEN a HyperShell deployment exists on OpenShift with baseline images diff --git a/specs/security/rbac-enforcement.spec.md b/specs/security/rbac-enforcement.spec.md index d7a8d55e3..5dfc9f6b4 100644 --- a/specs/security/rbac-enforcement.spec.md +++ b/specs/security/rbac-enforcement.spec.md @@ -413,7 +413,22 @@ when the caller has no binding that covers the requested resource. Returning 403 singleton GET leaks resource existence. For list endpoints, the middleware SHALL return 200 with an empty items array when the -caller has no matching resources. +caller has no matching resources. Collection `GET /gateways` SHALL be authorized for +any authenticated caller, including a user whose JWT carries only `hypershell-users` +and who has no `gateway:creator`, `platform:admin`, or per-gateway RoleBinding +(OpenShift `RBAC_DEFAULT_ROLES=`). The list handler then returns the empty collection. +Denying that list with 403 is a product bug: the web console treats it as +"Gateways could not be loaded". + +#### Scenario: Developer with no gateway bindings lists gateways + +- GIVEN production deployment with `RBAC_DEFAULT_ROLES=` +- AND user A is authenticated +- AND user A's JWT does not carry `gateway:creator` or `platform:admin` +- AND user A has no per-gateway RoleBinding +- WHEN user A calls `GET /api/hypershell/v1/gateways` +- THEN the response is 200 +- AND `items` is an empty array For mutation endpoints where the caller lacks write permission, the middleware SHALL return 403. diff --git a/tests/e2e/drivers/kind.sh b/tests/e2e/drivers/kind.sh index 99f9a22b3..8a7e1d5b3 100755 --- a/tests/e2e/drivers/kind.sh +++ b/tests/e2e/drivers/kind.sh @@ -223,19 +223,13 @@ discover_gateway_endpoint() { # audience mapper, and the gateway's Envoy validates aud == that client. A token # from the shared frontend client is rejected with InvalidAudience, so gateway and # CLI calls must mint tokens against the per-gateway client. -_driver_acquire_oidc_token() { - _OIDC_ACCESS_TOKEN="" - local username="${1:-${E2E_OIDC_USERNAME}}" - local password="${2:-${E2E_OIDC_PASSWORD}}" - local client_id="${3:-${E2E_OIDC_CLIENT_ID}}" - +# _driver_token_request - POST the given form fields to the realm token endpoint +# and set _OIDC_ACCESS_TOKEN from the access_token field. Every grant flow funnels +# through here so error handling and JSON parsing stay in one place. +_driver_token_request() { local token_endpoint="${E2E_OIDC_ISSUER}/protocol/openid-connect/token" local response - response=$(_driver_curl -X POST "${token_endpoint}" \ - -d "grant_type=password" \ - -d "client_id=${client_id}" \ - -d "username=${username}" \ - -d "password=${password}" 2>/dev/null || true) + response=$(_driver_curl -X POST "${token_endpoint}" "$@" 2>/dev/null || true) _OIDC_ACCESS_TOKEN=$(echo "$response" | python3 -c "import json,sys; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null || true) @@ -246,6 +240,78 @@ _driver_acquire_oidc_token() { fi } +# _driver_acquire_oidc_token - obtain an OIDC access token, honoring E2E_OIDC_GRANT +# (ephemeral-pr-environments.spec.md). Keeps the same [username] [password] +# [client_id] signature and call sites regardless of grant. +# +# password (default) -- resource-owner password grant against the seeded user. +# The Kind path and the default for manual OpenShift runs. +# client_credentials -- the GitHub-brokered pull-request path. Brokered GitHub +# users have no password grant, so tokens come from the confidential +# hypershell-e2e client instead: +# * admin HyperShell API token (username == E2E_OIDC_USERNAME and +# client_id == E2E_OIDC_CLIENT_ID) -- client-credentials on +# hypershell-e2e. +# * every other call, including acquire_gateway_token_with_role -- +# Keycloak token-exchange impersonating that principal for the +# requested audience. Never a password grant. +_driver_acquire_oidc_token() { + _OIDC_ACCESS_TOKEN="" + local username="${1:-${E2E_OIDC_USERNAME}}" + local password="${2:-${E2E_OIDC_PASSWORD}}" + local client_id="${3:-${E2E_OIDC_CLIENT_ID}}" + + case "${E2E_OIDC_GRANT:-password}" in + password) + _driver_token_request \ + -d "grant_type=password" \ + -d "client_id=${client_id}" \ + -d "username=${username}" \ + -d "password=${password}" + ;; + client_credentials) + if [[ -z "${E2E_OIDC_SA_CLIENT_SECRET:-}" ]]; then + red " E2E_OIDC_GRANT=client_credentials requires E2E_OIDC_SA_CLIENT_SECRET" + red " (the ${E2E_OIDC_SA_CLIENT_ID} client secret read from the Keycloak namespace after openshift-up)" + return 1 + fi + if [[ "${username}" == "${E2E_OIDC_USERNAME}" && "${client_id}" == "${E2E_OIDC_CLIENT_ID}" ]]; then + # Admin HyperShell API token: client-credentials on hypershell-e2e. + # Per-gateway tokens (a different client_id) must not use this path: + # the CC token is issued for hypershell-e2e / hypershell-frontend and + # never carries openshell-admin on the gateway client. + _driver_token_request \ + -d "grant_type=client_credentials" \ + -d "client_id=${E2E_OIDC_SA_CLIENT_ID}" \ + -d "client_secret=${E2E_OIDC_SA_CLIENT_SECRET}" + else + # Impersonate the requested principal targeting that audience. + # Keycloak 26 standard token-exchange rejects requested_subject; this + # is the legacy (token-exchange feature) impersonation grant and needs + # a subject_token (the hypershell-e2e client-credentials token). + if ! _driver_token_request \ + -d "grant_type=client_credentials" \ + -d "client_id=${E2E_OIDC_SA_CLIENT_ID}" \ + -d "client_secret=${E2E_OIDC_SA_CLIENT_SECRET}"; then + return 1 + fi + local subject_token="$_OIDC_ACCESS_TOKEN" + _driver_token_request \ + -d "grant_type=urn:ietf:params:oauth:grant-type:token-exchange" \ + -d "client_id=${E2E_OIDC_SA_CLIENT_ID}" \ + -d "client_secret=${E2E_OIDC_SA_CLIENT_SECRET}" \ + -d "subject_token=${subject_token}" \ + -d "requested_subject=${username}" \ + -d "audience=${client_id}" + fi + ;; + *) + red " Unknown E2E_OIDC_GRANT '${E2E_OIDC_GRANT}' (valid: password, client_credentials)" + return 1 + ;; + esac +} + acquire_oidc_token() { _driver_acquire_oidc_token "$@" } @@ -273,7 +339,7 @@ _patch_namespace_gc_timing() { return 1 fi _GC_TIMING_PATCHED=1 - if ! "$cli" rollout status deployment/hypershell-controller -n "$namespace" --timeout=120s >/dev/null; then + if ! "$cli" rollout status deployment/hypershell-controller -n "$namespace" --timeout=300s >/dev/null; then red " hypershell-controller did not roll out after GC timing patch" return 1 fi @@ -285,7 +351,7 @@ _restore_namespace_gc_timing() { dim " Restoring controller namespace GC timing to deployment defaults..." "$cli" set env deployment/hypershell-controller -n "$namespace" -c controller \ GATEWAY_NAMESPACE_GC_INTERVAL- GATEWAY_NAMESPACE_GC_GRACE_PERIOD- >/dev/null 2>&1 || true - "$cli" rollout status deployment/hypershell-controller -n "$namespace" --timeout=120s >/dev/null 2>&1 || true + "$cli" rollout status deployment/hypershell-controller -n "$namespace" --timeout=300s >/dev/null 2>&1 || true _GC_TIMING_PATCHED="" } @@ -463,13 +529,17 @@ PY # AssignClientRole bridge is asynchronous, so a token minted immediately after # gateway creation may not yet carry openshell-admin; poll until it does. # Sets _OIDC_ACCESS_TOKEN on success. +# +# Grant-agnostic: it delegates to acquire_oidc_token, so E2E_OIDC_GRANT selects +# the flow (password grant on Kind/manual OpenShift; token-exchange impersonation +# of the passed principal, targeting client_id, on the GitHub-brokered PR path). # Usage: acquire_gateway_token_with_role [timeout] acquire_gateway_token_with_role() { local username="${1:?username required}" local password="${2:?password required}" local client_id="${3:?client_id required}" local role="${4:?role required}" - local timeout="${5:-120}" + local timeout="${5:-300}" local deadline=$(($(date +%s) + timeout)) while [[ $(date +%s) -lt $deadline ]]; do @@ -511,7 +581,7 @@ wait_for_gateway_route() { local gw_name="${1:?gateway name required}" local gw_namespace="${2:?gateway namespace required}" - local timeout="${E2E_PROVISION_TIMEOUT:-180}" + local timeout="${E2E_PROVISION_TIMEOUT:-300}" local deadline=$(($(date +%s) + timeout)) dim " Waiting for Gateway route readiness (timeout: ${timeout}s)..." diff --git a/tests/e2e/drivers/openshift.sh b/tests/e2e/drivers/openshift.sh index b7ea7ba35..000687144 100644 --- a/tests/e2e/drivers/openshift.sh +++ b/tests/e2e/drivers/openshift.sh @@ -164,7 +164,7 @@ get_cli_binary() { wait_for_gateway_route() { local gw_name="${1:?gateway name required}" local gw_namespace="${2:?gateway namespace required}" - local timeout="${E2E_PROVISION_TIMEOUT:-180}" + local timeout="${E2E_PROVISION_TIMEOUT:-300}" local deadline=$(($(date +%s) + timeout)) dim " Waiting for Gateway route readiness (timeout: ${timeout}s)..." diff --git a/tests/e2e/e2e-openshell.sh b/tests/e2e/e2e-openshell.sh index a0dd07c6f..bfc9f6d72 100755 --- a/tests/e2e/e2e-openshell.sh +++ b/tests/e2e/e2e-openshell.sh @@ -21,16 +21,16 @@ # E2E_NAMESPACE Namespace for e2e resources (default: openshell-e2e) # E2E_GATEWAY_NAME Gateway name (default: e2e-gw-, unique per run) # E2E_MODE Run depth: long (default, every step) or short (essential steps) -# E2E_SANDBOX_TIMEOUT Seconds to wait for sandbox (default: 120) -# E2E_PROVISION_TIMEOUT Seconds to wait for gateway provisioning (default: 180) -# E2E_GC_TIMEOUT Seconds to wait for namespace GC after delete (default: 180) -# E2E_ORPHAN_GC_TIMEOUT Seconds to wait for periodic orphan namespace GC (default: 90) +# E2E_SANDBOX_TIMEOUT Seconds to wait for sandbox (default: 300) +# E2E_PROVISION_TIMEOUT Seconds to wait for gateway provisioning (default: 300) +# E2E_GC_TIMEOUT Seconds to wait for namespace GC after delete (default: 300) +# E2E_ORPHAN_GC_TIMEOUT Seconds to wait for periodic orphan namespace GC (default: 300) # E2E_SKIP_CLEANUP Set to 1 to keep test resources after run (default: 0) # DATABASE_PROVIDER Database provider: deployment, cnpg, or external (default: external) # E2E_CNPG_NAMESPACE Namespace where the CNPG operator runs (default: cnpg-system) # OPENSHELL_BIN Path to the openshell CLI binary (default: openshell) # E2E_OPENSHELL_INSTALL auto, always, or never (default: auto; CI uses always) -# E2E_GATEWAY_VERSION_TIMEOUT Seconds to wait for the runtime version (default: 120) +# E2E_GATEWAY_VERSION_TIMEOUT Seconds to wait for the runtime version (default: 300) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -1510,11 +1510,30 @@ except Exception: fi fi - # ── positive assertion: authenticated user receives gateway:creator by default ── - # RBAC_DEFAULT_ROLES defaults to gateway:creator, so every authenticated user - # is a creator. A developer with openshell-user Keycloak roles still gets the - # platform default binding and therefore can create gateways. This verifies - # that the default-role bootstrap fires correctly (HYPERSHELL-262). + # ── gateway list: collection GET must 200 even with no RoleBindings ── + # OpenShift RBAC_DEFAULT_ROLES= leaves developer with only hypershell-users. + # The list handler returns an empty items array; 403 is "Gateways could not + # be loaded" in the web console (rbac-enforcement Error Response Opacity). + show_cmd "curl ${API_HOST}/api/hypershell/v1/gateways (as developer) -> expect 200" + DEV_LIST_FILE=$(mktemp) + DEV_LIST_STATUS=$(_driver_curl -o "${DEV_LIST_FILE}" -w '%{http_code}' \ + "${API_HOST}/api/hypershell/v1/gateways" \ + -H "Authorization: Bearer ${DEV_TOKEN}" 2>/dev/null || true) + DEV_LIST_KIND=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("kind",""))' \ + "${DEV_LIST_FILE}" 2>/dev/null || true) + rm -f "${DEV_LIST_FILE}" + if [[ "${DEV_LIST_STATUS}" == "200" && "${DEV_LIST_KIND}" == "GatewayList" ]]; then + pass "Developer user: gateway list allowed (HTTP 200 GatewayList)" + else + fail_test "Developer user: gateway list returned HTTP ${DEV_LIST_STATUS:-none} kind=${DEV_LIST_KIND:-} (want 200 GatewayList)" + fi + + # ── gateway create: follows the deployment's RBAC_DEFAULT_ROLES ── + # Kind leaves RBAC_DEFAULT_ROLES unset, so the API default (gateway:creator) + # applies and every authenticated user can create (HYPERSHELL-262). OpenShift + # sets RBAC_DEFAULT_ROLES to empty (production isolation); developer is not a + # creator and MUST get 403 (e2e-testing.spec.md Openshell User May Not Create + # a Gateway). DEV_GW_CREATE_NAME="e2e-dev-gw-$(date +%s | tail -c5)" DEV_GW_BODY=$(GW_NAME="$DEV_GW_CREATE_NAME" E2E_OIDC_ISSUER="$E2E_OIDC_ISSUER" \ E2E_OIDC_CLIENT_ID="$E2E_OIDC_CLIENT_ID" python3 -c " @@ -1535,8 +1554,13 @@ body = { } print(json.dumps(body)) ") - show_cmd "curl -X POST ${API_HOST}/api/hypershell/v1/gateways (as developer) -> expect 201 (gateway:creator by default)" - dim " Expecting 201 Created (developer receives gateway:creator via RBAC_DEFAULT_ROLES)..." + if e2e_rbac_default_includes_creator; then + show_cmd "curl -X POST ${API_HOST}/api/hypershell/v1/gateways (as developer) -> expect 201 (gateway:creator by default)" + dim " Expecting 201 Created (developer receives gateway:creator via RBAC_DEFAULT_ROLES)..." + else + show_cmd "curl -X POST ${API_HOST}/api/hypershell/v1/gateways (as developer) -> expect 403 (no default gateway:creator)" + dim " Expecting 403 Forbidden (RBAC_DEFAULT_ROLES is empty; developer is not a creator)..." + fi DEV_GW_RESP_FILE=$(mktemp) DEV_GW_STATUS=$(_driver_curl -o "${DEV_GW_RESP_FILE}" -w '%{http_code}' \ @@ -1546,19 +1570,34 @@ print(json.dumps(body)) -d "${DEV_GW_BODY}" 2>/dev/null || true) DEV_GW_RESP=$(sed 's/\x1b\[[0-9;]*m//g' "${DEV_GW_RESP_FILE}" 2>/dev/null | tr '\n' ' ' | tr -s ' ') - if [[ "$DEV_GW_STATUS" =~ ^2 ]]; then - pass "Developer user: gateway create allowed (gateway:creator default binding active)" - DEV_DEFAULT_GW_ID=$(echo "$DEV_GW_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true) - if [[ -n "$DEV_DEFAULT_GW_ID" ]]; then - _driver_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${DEV_DEFAULT_GW_ID}" \ - -H "Authorization: Bearer ${DEV_TOKEN}" &>/dev/null || true + if e2e_rbac_default_includes_creator; then + if [[ "$DEV_GW_STATUS" =~ ^2 ]]; then + pass "Developer user: gateway create allowed (gateway:creator default binding active)" + DEV_DEFAULT_GW_ID=$(echo "$DEV_GW_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true) + if [[ -n "$DEV_DEFAULT_GW_ID" ]]; then + _driver_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${DEV_DEFAULT_GW_ID}" \ + -H "Authorization: Bearer ${DEV_TOKEN}" &>/dev/null || true + fi + elif [[ "$DEV_GW_STATUS" == "403" ]]; then + fail_test "Developer user: gateway create blocked -- default gateway:creator binding was not assigned (HTTP 403)" + dim " ${DEV_GW_RESP:0:200}" + else + fail_test "Developer user: unexpected HTTP ${DEV_GW_STATUS:-none} on gateway create" + dim " ${DEV_GW_RESP:0:200}" fi - elif [[ "$DEV_GW_STATUS" == "403" ]]; then - fail_test "Developer user: gateway create blocked -- default gateway:creator binding was not assigned (HTTP 403)" - dim " ${DEV_GW_RESP:0:200}" else - fail_test "Developer user: unexpected HTTP ${DEV_GW_STATUS:-none} on gateway create" - dim " ${DEV_GW_RESP:0:200}" + if [[ "$DEV_GW_STATUS" == "403" ]]; then + pass "Developer user: gateway create denied (HTTP 403, no default gateway:creator)" + elif [[ "$DEV_GW_STATUS" =~ ^2 ]]; then + fail_test "Developer user: gateway create succeeded -- RBAC_DEFAULT_ROLES is empty so this must be 403" + DEV_DEFAULT_GW_ID=$(echo "$DEV_GW_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true) + if [[ -n "$DEV_DEFAULT_GW_ID" ]]; then + api_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${DEV_DEFAULT_GW_ID}" &>/dev/null || true + fi + else + fail_test "Developer user: unexpected HTTP ${DEV_GW_STATUS:-none} on gateway create" + dim " ${DEV_GW_RESP:0:200}" + fi fi rm -f "${DEV_GW_RESP_FILE}" 2>/dev/null || true @@ -1664,10 +1703,9 @@ print('true' if has_owner else 'false') fi rm -f "${PADMIN_DELETE_FILE}" 2>/dev/null || true - # ── positive assertion: platform:admin also receives gateway:creator by default ── - # RBAC_DEFAULT_ROLES applies to all authenticated users including platform:admin. - # They can create gateways via the default binding even without explicit - # gateway:creator in their Keycloak realm roles (HYPERSHELL-262). + # ── gateway create: platform:admin is view and delete, not create ── + # Kind's default RBAC_DEFAULT_ROLES still grants gateway:creator (HYPERSHELL-262). + # OpenShift leaves that env empty, so this POST MUST be 403. PADMIN_GW_CREATE_NAME="e2e-padmin-gw-$(date +%s | tail -c5)" PADMIN_GW_BODY=$(GW_NAME="$PADMIN_GW_CREATE_NAME" E2E_OIDC_ISSUER="$E2E_OIDC_ISSUER" \ E2E_OIDC_CLIENT_ID="$E2E_OIDC_CLIENT_ID" python3 -c " @@ -1688,8 +1726,13 @@ body = { } print(json.dumps(body)) ") - show_cmd "curl -X POST ${API_HOST}/api/hypershell/v1/gateways (as platform admin) -> expect 201 (gateway:creator by default)" - dim " Expecting 201 Created (platform:admin receives gateway:creator via RBAC_DEFAULT_ROLES)..." + if e2e_rbac_default_includes_creator; then + show_cmd "curl -X POST ${API_HOST}/api/hypershell/v1/gateways (as platform admin) -> expect 201 (gateway:creator by default)" + dim " Expecting 201 Created (platform:admin receives gateway:creator via RBAC_DEFAULT_ROLES)..." + else + show_cmd "curl -X POST ${API_HOST}/api/hypershell/v1/gateways (as platform admin) -> expect 403 (no default gateway:creator)" + dim " Expecting 403 Forbidden (platform:admin is view and delete; create needs gateway:creator)..." + fi PADMIN_CREATE_FILE=$(mktemp) PADMIN_CREATE_STATUS=$(_driver_curl -o "${PADMIN_CREATE_FILE}" -w '%{http_code}' \ @@ -1699,19 +1742,34 @@ print(json.dumps(body)) -d "${PADMIN_GW_BODY}" 2>/dev/null || true) PADMIN_CREATE_RESP=$(cat "${PADMIN_CREATE_FILE}" 2>/dev/null || true) - if [[ "$PADMIN_CREATE_STATUS" =~ ^2 ]]; then - pass "Platform admin: gateway create allowed (gateway:creator default binding active)" - PADMIN_DEFAULT_GW_ID=$(echo "$PADMIN_CREATE_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true) - if [[ -n "$PADMIN_DEFAULT_GW_ID" ]]; then - _driver_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${PADMIN_DEFAULT_GW_ID}" \ - -H "Authorization: Bearer ${PADMIN_TOKEN}" &>/dev/null || true + if e2e_rbac_default_includes_creator; then + if [[ "$PADMIN_CREATE_STATUS" =~ ^2 ]]; then + pass "Platform admin: gateway create allowed (gateway:creator default binding active)" + PADMIN_DEFAULT_GW_ID=$(echo "$PADMIN_CREATE_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true) + if [[ -n "$PADMIN_DEFAULT_GW_ID" ]]; then + _driver_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${PADMIN_DEFAULT_GW_ID}" \ + -H "Authorization: Bearer ${PADMIN_TOKEN}" &>/dev/null || true + fi + elif [[ "$PADMIN_CREATE_STATUS" == "403" ]]; then + fail_test "Platform admin: gateway create blocked -- default gateway:creator binding was not assigned (HTTP 403)" + dim " ${PADMIN_CREATE_RESP:0:200}" + else + fail_test "Platform admin: unexpected HTTP ${PADMIN_CREATE_STATUS:-none} on gateway create" + dim " ${PADMIN_CREATE_RESP:0:200}" fi - elif [[ "$PADMIN_CREATE_STATUS" == "403" ]]; then - fail_test "Platform admin: gateway create blocked -- default gateway:creator binding was not assigned (HTTP 403)" - dim " ${PADMIN_CREATE_RESP:0:200}" else - fail_test "Platform admin: unexpected HTTP ${PADMIN_CREATE_STATUS:-none} on gateway create" - dim " ${PADMIN_CREATE_RESP:0:200}" + if [[ "$PADMIN_CREATE_STATUS" == "403" ]]; then + pass "Platform admin: gateway create denied (HTTP 403, no default gateway:creator)" + elif [[ "$PADMIN_CREATE_STATUS" =~ ^2 ]]; then + fail_test "Platform admin: gateway create succeeded -- RBAC_DEFAULT_ROLES is empty so this must be 403" + PADMIN_DEFAULT_GW_ID=$(echo "$PADMIN_CREATE_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || true) + if [[ -n "$PADMIN_DEFAULT_GW_ID" ]]; then + api_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${PADMIN_DEFAULT_GW_ID}" &>/dev/null || true + fi + else + fail_test "Platform admin: unexpected HTTP ${PADMIN_CREATE_STATUS:-none} on gateway create" + dim " ${PADMIN_CREATE_RESP:0:200}" + fi fi rm -f "${PADMIN_CREATE_FILE}" 2>/dev/null || true fi diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh index 79558134a..f3c877f59 100755 --- a/tests/e2e/lib.sh +++ b/tests/e2e/lib.sh @@ -149,11 +149,12 @@ e2e_validate_openshell_install() { : "${E2E_NAMESPACE:=openshell-e2e}" : "${E2E_GATEWAY_NAME:=e2e-gw-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')}" : "${E2E_MODE:=long}" -: "${E2E_SANDBOX_TIMEOUT:=120}" -: "${E2E_PROVISION_TIMEOUT:=180}" -: "${E2E_GC_TIMEOUT:=180}" -: "${E2E_ORPHAN_GC_TIMEOUT:=90}" +: "${E2E_SANDBOX_TIMEOUT:=300}" +: "${E2E_PROVISION_TIMEOUT:=300}" +: "${E2E_GC_TIMEOUT:=300}" +: "${E2E_ORPHAN_GC_TIMEOUT:=300}" : "${E2E_SKIP_CLEANUP:=0}" +: "${E2E_AUTO_SEED:=1}" : "${E2E_PAUSE:=1}" _E2E_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" : "${OPENSHELL_BIN:=openshell}" @@ -167,7 +168,7 @@ _E2E_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" : "${OPENSHELL_INSTALL_SCRIPT_URL:=https://raw.githubusercontent.com/openshift-online/hypershell/main/scripts/install-openshell.sh}" # Bounded wait for the control plane to reconcile gateway_version from the # gateway's health endpoint before deriving the install command. -: "${E2E_GATEWAY_VERSION_TIMEOUT:=120}" +: "${E2E_GATEWAY_VERSION_TIMEOUT:=300}" : "${E2E_KEYCLOAK_NAMESPACE:=keycloak}" : "${E2E_OIDC_ISSUER:=https://keycloak.hypershell.localhost/realms/hypershell}" : "${E2E_OIDC_CLIENT_ID:=hypershell-frontend}" @@ -183,6 +184,19 @@ _E2E_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" : "${E2E_KC_ADMIN_USER:=admin}" : "${E2E_KC_ADMIN_PASSWORD:=admin}" +# Token grant for acquire_oidc_token / acquire_gateway_token_with_role, per +# ephemeral-pr-environments.spec.md (HYPERSHELL-240). "password" (default) is the +# Kind and manual OpenShift path: a resource-owner password grant against seeded +# users. "client_credentials" is the GitHub-brokered pull-request path -- brokered +# GitHub users have no password grant, so CI authenticates through the confidential +# hypershell-e2e service-account client (client-credentials for the admin path) and +# Keycloak token exchange (impersonating the seeded developer principal). CI reads +# the hypershell-e2e secret from the deployed Keycloak namespace and exports it as +# E2E_OIDC_SA_CLIENT_SECRET; it never comes from a repo secret. +: "${E2E_OIDC_GRANT:=password}" +: "${E2E_OIDC_SA_CLIENT_ID:=hypershell-e2e}" +: "${E2E_OIDC_SA_CLIENT_SECRET:=}" + # RFC3339 timestamp N minutes in the past (macOS BSD date and GNU date). e2e_gc_eligible_since_backdate() { local minutes="${1:-3}" @@ -299,14 +313,116 @@ try: data = json.load(sys.stdin) except Exception: sys.exit(0) -items = data.get('items', []) if isinstance(data, dict) else [] +if isinstance(data, dict): + if data.get('kind') == 'Error': + sys.exit(0) + items = data.get('items') or [] +elif isinstance(data, list): + items = data +else: + items = [] for it in items: - if not name or it.get('name', '') == name: + if isinstance(it, dict) and (not name or it.get('name', '') == name): print(it.get('id', '') or '') break " } +# One-line summary of a HyperShell list (or error) JSON body on stdin. +# Distinguishes empty lists from 401/403 Error payloads and unparseable bodies +# so seed-discovery failures are not just "id=". +e2e_json_list_summary() { + python3 -c ' +import json, sys +raw = sys.stdin.read() +if not raw.strip(): + print("empty-body") + raise SystemExit(0) +try: + data = json.loads(raw) +except Exception: + print("unparseable") + raise SystemExit(0) +if isinstance(data, list): + print("kind= items=%s" % len(data)) + raise SystemExit(0) +if not isinstance(data, dict): + print("non-object") + raise SystemExit(0) +if data.get("kind") == "Error": + print("error code=%s reason=%s" % (data.get("code") or "", data.get("reason") or "")) + raise SystemExit(0) +items = data.get("items") or [] +if not isinstance(items, list): + items = [] +total = data.get("total") +total_s = "" if total is None else total +print("kind=%s total=%s items=%s" % (data.get("kind") or "", total_s, len(items))) +' +} + +# Effective RBAC_DEFAULT_ROLES from an api-server container env JSON array +# (kubectl/oc jsonpath of .spec.template.spec.containers[?(@.name=="api-server")].env). +# Unset matches the Go default (gateway:creator). An explicit empty value is +# the OpenShift/production isolation posture and must not be treated as unset. +e2e_effective_rbac_default_roles_from_env_json() { + python3 -c ' +import json, sys +raw = sys.stdin.read().strip() +if not raw: + print("gateway:creator") + raise SystemExit(0) +try: + env = json.loads(raw) +except Exception: + print("gateway:creator") + raise SystemExit(0) +if not isinstance(env, list): + print("gateway:creator") + raise SystemExit(0) +for item in env: + if isinstance(item, dict) and item.get("name") == "RBAC_DEFAULT_ROLES": + print(item.get("value") or "") + raise SystemExit(0) +print("gateway:creator") +' +} + +e2e_read_api_server_container_env() { + local cli="${CLI:-kubectl}" + local ns="${E2E_HS_NAMESPACE:-hypershell-system}" + "$cli" get deployment hypershell-api-server -n "$ns" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="api-server")].env}' \ + 2>/dev/null || true +} + +e2e_effective_rbac_default_roles() { + local raw + raw="$(e2e_read_api_server_container_env)" + if [[ -z "${raw}" ]]; then + if [[ "${E2E_INFRA_DRIVER:-}" == "openshift" ]]; then + printf '' + return 0 + fi + printf '%s' 'gateway:creator' + return 0 + fi + printf '%s' "${raw}" | e2e_effective_rbac_default_roles_from_env_json +} + +e2e_rbac_default_includes_creator() { + if [[ -z "${_E2E_RBAC_DEFAULT_INCLUDES_CREATOR:-}" ]]; then + local roles + roles="$(e2e_effective_rbac_default_roles)" + if [[ ",${roles}," == *",gateway:creator,"* ]]; then + _E2E_RBAC_DEFAULT_INCLUDES_CREATOR=yes + else + _E2E_RBAC_DEFAULT_INCLUDES_CREATOR=no + fi + fi + [[ "${_E2E_RBAC_DEFAULT_INCLUDES_CREATOR}" == "yes" ]] +} + # Look up a gateway by exact name. Sets _GW_ID, _GW_NAMESPACE, _GW_PHASE (empty if missing). # Requires API_HOST and api_curl. e2e_lookup_gateway_by_name() { @@ -338,14 +454,65 @@ else: # # Name pins (optional): E2E_SEED_CLUSTER_NAME, E2E_SEED_RELEASE_NAME. # On kind these default to the make kind-up seeds (local-kind, dev-release). -# When a name is unset, the first list item is used - that matches +# On openshift they default to the make openshift-seed names (local-openshift, +# dev-release). When a name is unset, the first list item is used - that matches # single-seed CI/dev; multi-seed clusters should set the name pins instead # of relying on API order. -e2e_discover_seed_ids() { +# +# When both cluster and release lists are empty collections (not an API Error), +# and E2E_AUTO_SEED is not 0, discovery runs `SEED_STRICT=true make -seed` +# once and retries. That recovers a PR env whose last openshift-up wiped the +# database and failed before the seed step. +e2e_summary_is_empty_list() { + [[ "${1:-}" == kind=*List* && "${1:-}" == *"items=0"* ]] +} + +e2e_inventory_unseeded() { + e2e_summary_is_empty_list "${_E2E_CLUSTER_LIST_SUMMARY:-}" \ + && e2e_summary_is_empty_list "${_E2E_RELEASE_LIST_SUMMARY:-}" +} + +e2e_auto_seed_enabled() { + case "${E2E_AUTO_SEED:-1}" in + 0|false|FALSE|no|NO) return 1 ;; + *) return 0 ;; + esac +} + +e2e_run_platform_seed() { + local root + root="$(cd "${_E2E_LIB_DIR}/../.." && pwd)" + case "${E2E_INFRA_DRIVER:-}" in + kind) + (cd "${root}" && SEED_STRICT=true make kind-seed) + ;; + openshift) + (cd "${root}" && SEED_STRICT=true make openshift-seed) + ;; + *) + red "ERROR: no platform seed target for driver '${E2E_INFRA_DRIVER:-}'" + return 1 + ;; + esac +} + +e2e_print_seed_discovery_error() { + red "ERROR: could not discover seeded cluster/release ids from the API" + dim " cluster=${E2E_SEED_CLUSTER_NAME:-} id=${E2E_CLUSTER_ID:-} (${_E2E_CLUSTER_LIST_SUMMARY:-unknown})" + dim " release=${E2E_SEED_RELEASE_NAME:-} id=${E2E_RELEASE_ID:-} (${_E2E_RELEASE_LIST_SUMMARY:-unknown})" + dim " database=${E2E_DATABASE_ID:-} (${_E2E_DATABASE_LIST_SUMMARY:-unknown})" + dim " Re-seed once the API is healthy: SEED_STRICT=true make openshift-seed" + dim " (Kind: SEED_STRICT=true make kind-seed)" +} + +e2e_fetch_seed_ids() { local clusters releases databases if [[ "${E2E_INFRA_DRIVER:-}" == "kind" ]]; then : "${E2E_SEED_CLUSTER_NAME:=local-kind}" : "${E2E_SEED_RELEASE_NAME:=dev-release}" + elif [[ "${E2E_INFRA_DRIVER:-}" == "openshift" ]]; then + : "${E2E_SEED_CLUSTER_NAME:=local-openshift}" + : "${E2E_SEED_RELEASE_NAME:=dev-release}" else : "${E2E_SEED_CLUSTER_NAME:=}" : "${E2E_SEED_RELEASE_NAME:=}" @@ -358,13 +525,30 @@ e2e_discover_seed_ids() { E2E_CLUSTER_ID=$(echo "$clusters" | e2e_json_first_id "${E2E_SEED_CLUSTER_NAME}") E2E_RELEASE_ID=$(echo "$releases" | e2e_json_first_id "${E2E_SEED_RELEASE_NAME}") E2E_DATABASE_ID=$(echo "$databases" | e2e_json_first_id) + _E2E_CLUSTER_LIST_SUMMARY=$(echo "$clusters" | e2e_json_list_summary) + _E2E_RELEASE_LIST_SUMMARY=$(echo "$releases" | e2e_json_list_summary) + _E2E_DATABASE_LIST_SUMMARY=$(echo "$databases" | e2e_json_list_summary) - if [[ -z "${E2E_CLUSTER_ID}" || -z "${E2E_RELEASE_ID}" ]]; then - red "ERROR: could not discover seeded cluster/release ids from the API" - dim " cluster=${E2E_SEED_CLUSTER_NAME:-} id=${E2E_CLUSTER_ID:-}" - dim " release=${E2E_SEED_RELEASE_NAME:-} id=${E2E_RELEASE_ID:-}" - return 1 + e2e_seed_ids_ready +} + +e2e_discover_seed_ids() { + if e2e_fetch_seed_ids; then + return 0 fi + if e2e_inventory_unseeded && e2e_auto_seed_enabled; then + dim " Seed inventory is empty; running SEED_STRICT=true make ${E2E_INFRA_DRIVER}-seed..." + if ! e2e_run_platform_seed; then + red "ERROR: platform seed failed; cannot discover cluster/release ids" + e2e_print_seed_discovery_error + return 1 + fi + if e2e_fetch_seed_ids; then + return 0 + fi + fi + e2e_print_seed_discovery_error + return 1 } e2e_seed_ids_ready() { diff --git a/tests/e2e/openshift_driver_test.sh b/tests/e2e/openshift_driver_test.sh index e8792672c..2232e7e33 100644 --- a/tests/e2e/openshift_driver_test.sh +++ b/tests/e2e/openshift_driver_test.sh @@ -36,7 +36,7 @@ oc() { *"get gateway shared-gateway -n openshift-ingress"*) printf '%s\n' 'Programmed=True' ;; *"get grpcroute openshell-gateway -n tenant-a"*) printf '%s\n' 'Accepted=True' ;; *"set env deployment/hypershell-controller -n test-team -c controller GATEWAY_NAMESPACE_GC_INTERVAL=30s GATEWAY_NAMESPACE_GC_GRACE_PERIOD=30s"*) : ;; - *"rollout status deployment/hypershell-controller -n test-team --timeout=120s"*) : ;; + *"rollout status deployment/hypershell-controller -n test-team --timeout=300s"*) : ;; *"set env deployment/hypershell-controller -n test-team -c controller GATEWAY_NAMESPACE_GC_INTERVAL- GATEWAY_NAMESPACE_GC_GRACE_PERIOD-"*) : ;; *) return 1 ;; esac @@ -98,5 +98,68 @@ else fi assert_eq '' "${_GC_TIMING_PATCHED}" 'namespace GC timing patch cleared after restore' +# --- E2E_OIDC_GRANT dispatch (ephemeral-pr-environments.spec.md, PR-ENV-10) --- +# Replace the reachability stub with a token-endpoint stub that captures the POST +# body and returns a JSON access token so acquire_oidc_token parses successfully. +E2E_OIDC_ISSUER='https://sso-test.apps.example.com/realms/hypershell' +E2E_OIDC_USERNAME='admin' +E2E_OIDC_PASSWORD='admin' +E2E_OIDC_CLIENT_ID='hypershell-frontend' +E2E_OIDC_SA_CLIENT_ID='hypershell-e2e' +E2E_OIDC_SA_CLIENT_SECRET='s3cr3t' +# The production token request runs curl inside $(...), a subshell, so capture the +# request body through a file that survives the subshell rather than a variable. +CURL_CAPTURE="$(mktemp)" +curl() { + printf '%s' "$*" >"${CURL_CAPTURE}" + printf '%s' '{"access_token":"stub.jwt.token"}' +} +captured_curl_args() { printf ' %s ' "$(cat "${CURL_CAPTURE}")"; } + +# Default grant is the resource-owner password grant against the seeded user. +E2E_OIDC_GRANT=password acquire_oidc_token >/dev/null +case "$(captured_curl_args)" in + *' grant_type=password '*' client_id=hypershell-frontend '*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); printf 'FAIL: password grant args (got=%q)\n' "$(captured_curl_args)" ;; +esac + +# Admin client_credentials path uses the hypershell-e2e service-account client. +E2E_OIDC_GRANT=client_credentials acquire_oidc_token >/dev/null +case "$(captured_curl_args)" in + *' grant_type=client_credentials '*' client_id=hypershell-e2e '*' client_secret=s3cr3t '*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); printf 'FAIL: client_credentials admin args (got=%q)\n' "$(captured_curl_args)" ;; +esac + +# Developer path: client-credentials subject_token, then legacy impersonation +# (requested_subject). Keycloak 26 standard token-exchange rejects that param. +E2E_OIDC_GRANT=client_credentials acquire_oidc_token developer developer openshell-gw-1 >/dev/null +case "$(captured_curl_args)" in + *'grant_type=urn:ietf:params:oauth:grant-type:token-exchange'*' subject_token=stub.jwt.token '*' requested_subject=developer '*' audience=openshell-gw-1 '*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); printf 'FAIL: token-exchange developer args (got=%q)\n' "$(captured_curl_args)" ;; +esac + +# Admin + per-gateway client must also token-exchange. Straight client_credentials +# on hypershell-e2e never carries openshell-admin for that gateway client +# (e2e-testing.spec.md acquire_gateway_token_with_role). +E2E_OIDC_GRANT=client_credentials acquire_oidc_token admin admin openshell-gw-1 >/dev/null +case "$(captured_curl_args)" in + *'grant_type=urn:ietf:params:oauth:grant-type:token-exchange'*' subject_token=stub.jwt.token '*' requested_subject=admin '*' audience=openshell-gw-1 '*) PASS=$((PASS + 1)) ;; + *) FAIL=$((FAIL + 1)); printf 'FAIL: token-exchange admin gateway args (got=%q)\n' "$(captured_curl_args)" ;; +esac + +# client_credentials without the service-account secret fails fast. +if (E2E_OIDC_GRANT=client_credentials E2E_OIDC_SA_CLIENT_SECRET='' acquire_oidc_token >/dev/null 2>&1); then + FAIL=$((FAIL + 1)); echo 'FAIL: client_credentials without secret was accepted' +else + PASS=$((PASS + 1)) +fi + +# An unrecognized grant is rejected rather than silently defaulting. +if (E2E_OIDC_GRANT=totp acquire_oidc_token >/dev/null 2>&1); then + FAIL=$((FAIL + 1)); echo 'FAIL: unknown E2E_OIDC_GRANT was accepted' +else + PASS=$((PASS + 1)) +fi + printf 'OpenShift driver tests: %d passed, %d failed\n' "$PASS" "$FAIL" [[ "$FAIL" -eq 0 ]] diff --git a/tests/e2e/perf/lib_test.sh b/tests/e2e/perf/lib_test.sh index 68f29893b..e07b570e3 100755 --- a/tests/e2e/perf/lib_test.sh +++ b/tests/e2e/perf/lib_test.sh @@ -106,6 +106,79 @@ else fail_u "e2e_json_first_id name/first unexpected: picked=${picked} first=${first}" fi +err_id=$(echo '{"kind":"Error","id":"9","code":"403","reason":"Forbidden"}' | e2e_json_first_id) +if [[ -z "$err_id" ]]; then + pass_u "e2e_json_first_id ignores Error payloads" +else + fail_u "e2e_json_first_id should ignore Error ids, got ${err_id}" +fi + +sum_ok=$(echo '{"kind":"ManagedClusterList","total":1,"items":[{"id":"c1","name":"local-openshift"}]}' | e2e_json_list_summary) +sum_empty=$(echo '{"kind":"ManagedClusterList","total":0,"items":[]}' | e2e_json_list_summary) +sum_err=$(echo '{"kind":"Error","code":"403","reason":"Forbidden"}' | e2e_json_list_summary) +sum_blank=$(printf '' | e2e_json_list_summary) +if [[ "$sum_ok" == "kind=ManagedClusterList total=1 items=1" \ + && "$sum_empty" == "kind=ManagedClusterList total=0 items=0" \ + && "$sum_err" == "error code=403 reason=Forbidden" \ + && "$sum_blank" == "empty-body" ]]; then + pass_u "e2e_json_list_summary distinguishes lists, empty lists, and Error bodies" +else + fail_u "e2e_json_list_summary unexpected: ok=${sum_ok} empty=${sum_empty} err=${sum_err} blank=${sum_blank}" +fi + +_orig_api_curl=$(declare -f api_curl || true) +api_curl() { + case "$1" in + *managed_clusters) printf '%s' '{"kind":"ManagedClusterList","total":1,"items":[{"id":"c-os","name":"local-openshift"}]}' ;; + *gateway_releases) printf '%s' '{"kind":"GatewayReleaseList","total":1,"items":[{"id":"r-os","name":"dev-release"}]}' ;; + *managed_databases) printf '%s' '{"kind":"ManagedDatabaseList","total":1,"items":[{"id":"d-os","name":"openshell-db"}]}' ;; + *) printf '%s' '{"kind":"Error","reason":"unexpected url"}' ;; + esac +} +_saved_seed_cluster="${E2E_SEED_CLUSTER_NAME-}" +_saved_seed_release="${E2E_SEED_RELEASE_NAME-}" +unset E2E_SEED_CLUSTER_NAME E2E_SEED_RELEASE_NAME +E2E_INFRA_DRIVER=openshift API_HOST=https://example.invalid +E2E_CLUSTER_ID="" E2E_RELEASE_ID="" E2E_DATABASE_ID="" +if e2e_discover_seed_ids \ + && [[ "$E2E_CLUSTER_ID" == "c-os" && "$E2E_RELEASE_ID" == "r-os" && "$E2E_SEED_CLUSTER_NAME" == "local-openshift" ]]; then + pass_u "OpenShift discovery pins local-openshift / dev-release" +else + fail_u "OpenShift discovery pin failed: cluster=${E2E_CLUSTER_ID:-} release=${E2E_RELEASE_ID:-} name=${E2E_SEED_CLUSTER_NAME:-}" +fi + +api_curl() { + printf '%s' '{"kind":"Error","code":"403","reason":"Forbidden"}' +} +unset E2E_SEED_CLUSTER_NAME E2E_SEED_RELEASE_NAME +E2E_INFRA_DRIVER=openshift +E2E_CLUSTER_ID="" E2E_RELEASE_ID="" E2E_DATABASE_ID="" +disc_err="$(e2e_discover_seed_ids 2>&1 || true)" +if [[ "$disc_err" == *"error code=403 reason=Forbidden"* && "$disc_err" == *"make openshift-seed"* ]]; then + pass_u "seed discovery failure names Error payloads and the re-seed hint" +else + fail_u "seed discovery failure diagnostics missing: ${disc_err}" +fi +if [[ -n "${_orig_api_curl}" ]]; then + eval "${_orig_api_curl}" +else + unset -f api_curl +fi +unset _orig_api_curl +if [[ -n "${_saved_seed_cluster}" ]]; then + E2E_SEED_CLUSTER_NAME="${_saved_seed_cluster}" +else + unset E2E_SEED_CLUSTER_NAME +fi +if [[ -n "${_saved_seed_release}" ]]; then + E2E_SEED_RELEASE_NAME="${_saved_seed_release}" +else + unset E2E_SEED_RELEASE_NAME +fi +unset _saved_seed_cluster _saved_seed_release +E2E_CLUSTER_ID="" E2E_RELEASE_ID="" E2E_DATABASE_ID="" +unset E2E_INFRA_DRIVER API_HOST + # --- Percentiles (nearest-rank: ceil(p/100*n) for 1..10 -> 5, 9, 10, 10) --- json=$(perf_percentiles_json 1 2 3 4 5 6 7 8 9 10) @@ -400,6 +473,22 @@ else fail_u "e2e-performance.sh or e2e-openshell.sh does not call e2e_select_infra_driver" fi +if grep -q 'e2e_rbac_default_includes_creator' "${SCRIPT_DIR}/../e2e-openshell.sh"; then + pass_u "e2e-openshell.sh gates creator POST on the deployment RBAC_DEFAULT_ROLES" +else + fail_u "e2e-openshell.sh does not call e2e_rbac_default_includes_creator" +fi + +unset _E2E_RBAC_DEFAULT_INCLUDES_CREATOR +kind_roles=$(echo '[{"name":"RBAC_ENFORCE","value":"true"}]' | e2e_effective_rbac_default_roles_from_env_json) +os_roles=$(echo '[{"name":"RBAC_ENFORCE","value":"true"},{"name":"RBAC_DEFAULT_ROLES","value":""}]' | e2e_effective_rbac_default_roles_from_env_json) +explicit_roles=$(echo '[{"name":"RBAC_DEFAULT_ROLES","value":"gateway:creator"}]' | e2e_effective_rbac_default_roles_from_env_json) +if [[ "$kind_roles" == "gateway:creator" && -z "$os_roles" && "$explicit_roles" == "gateway:creator" ]]; then + pass_u "RBAC_DEFAULT_ROLES unset is gateway:creator; explicit empty stays empty" +else + fail_u "RBAC_DEFAULT_ROLES parse unexpected: kind=${kind_roles} os=${os_roles:-} explicit=${explicit_roles}" +fi + if grep -q 'E2E_INFRA_DRIVER is not set' "${SCRIPT_DIR}/../e2e-performance.sh"; then fail_u "e2e-performance.sh still requires E2E_INFRA_DRIVER to be set" else