From 80effdbe5eb6348a4a9905c577fe0dd4d40eddb7 Mon Sep 17 00:00:00 2001 From: CHAI Bot Date: Mon, 28 Sep 2026 14:10:29 +0000 Subject: [PATCH] Boilerplate: Update to 1191e12968520575f7177f1ff055e966b91cd52a Conventions: - openshift/golang-osd-e2e: Update --- https://github.com/openshift/boilerplate/compare/195c29d18279f69ea5b4e53ae9d90a795c27144a...1191e12968520575f7177f1ff055e966b91cd52a commit: 03034cff791e779ddf1ee2f5d2464ac491ffc0b3 author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi9 docker tag to v9.8-1790556197 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 83ce3d1e35295d32128733eaadfaa5072ffefe39 author: red-hat-konflux[bot] chore(deps): update konflux references to v0.12.3 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 2544f544ff4a01e0357ea1bdff006df559d93d73 author: Chai Bot ROSAENG-65653: Fix Impersonate scheme loss and KUBECONFIG multi-path handling Bug 1: Impersonate() called NewE2EClientFromConfig without the original WithScheme options, causing the impersonated client to lose operator- specific CRD types. Fix: store extraSchemes on E2EClient and forward them when creating the impersonated client. Bug 2: loadKubeConfig() passed the raw KUBECONFIG env var to BuildConfigFromFlags, which treats multi-path values (e.g. "/a:/b") as a single filename. Fix: use client-go's NewDefaultClientConfigLoadingRules which properly handles multi-path KUBECONFIG and falls back to ~/.kube/config. Co-Authored-By: Claude Opus 4.6 commit: 69382a9102db56fd1c2a619f4c2b808668bc9660 author: Chai Bot golang-osd-e2e: security and correctness fixes Addresses CodeRabbit review feedback from consumer repos: - Dockerfile: add non-root USER, use targeted COPY, layer go mod download - e2e-template.yml: add readOnlyRootFilesystem, writable emptyDir mounts - gangway-bridge-template.yml: fix backoff overflow, deadline check ordering, JOB_ENVS parsing, timeout retry behavior - README.md (generated): fix Ginkgo v2 install path, restrict kubeconfig perms commit: 3154cbd3de7885187b959f369309e757d88f24ee author: Chai Bot Add generalized e2eClient helper to golang-osd-e2e convention Extract and generalize the e2eClient helper from cloud-ingress-operator (PR #523) into the boilerplate convention so all OSD operators can share it. Key design decisions: - Exported E2EClient type with functional options pattern - WithScheme(addToScheme) hook lets each operator register its own CRD types without editing the boilerplate-owned file - Base scheme always includes core/v1, apps/v1, and openshift/api - Cluster metadata helpers (IsSTS, GetProvider, GetRegion) included - update script copies the file on every boilerplate-update ROSAENG-65653 Co-Authored-By: Claude Opus 4.6 commit: f6b885b3fa690ec45d0a26925d71ff6964821142 author: red-hat-konflux[bot] chore(deps): update konflux references Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: bc807b4dca754c158a1429ccd4718760c1ede3f6 author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi8/ubi-minimal:latest docker digest to 93288f4 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 5f7546e7c749ae0002a63f8e1c6a176782d9ee81 author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi9 docker tag to v9.8-1790067847 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: f8f62285f56fbeb308ae9c5f7741a2a3ef0fcc84 author: Josh Branham Update tag-dockerfile-changes.yml commit: a2f1925f325fdb7c71ac78715e9111b568ff7229 author: Josh Branham Create GitHub releases for image tags commit: 9df55ef478d46a83f0dd2e30d1c4acb863fd9538 author: Josh Branham Add daily Dockerfile image tagging workflow commit: 8ac781ff7387a6c123f97c1e1cb95f6123771c15 author: Josh Branham Use latest boilerplate image tag for consumers commit: 3b6169c85ea0f6e54ec647a1bc89093793860250 author: red-hat-konflux[bot] chore(deps): update konflux references Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 7181cfe1300fe7dfabe5ddecb163b16291d2e4ed author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi9 docker tag to v9.8-1789646010 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 9a85efd2321b01bb5bdae15a865b3f707aead488 author: Josh Branham Configure Dockerfile Renovate updates for Prow commit: ee00013ea57f3a13a9d4a3455fe62c5b49018575 author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi8/ubi-minimal:latest docker digest to 1280211 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 1a3e095b472dccaac658c4e53dc42db95fc997b4 author: red-hat-konflux[bot] chore(deps): update konflux references Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 9096aef5786812df4e8b4c51fc1da5164b49fb17 author: red-hat-konflux[bot] chore(deps): pin quay.io/konflux-ci/yq docker tag to 9b73d39 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 48a1701b9a66a1c0d60c7450e719f73fceec9315 author: Josh Branham Delete .tekton/image-push.yaml commit: ccfdd269c125f472548674ae7465da61f53c195f author: Josh Branham ROSAENG-67043: Add auto-release pipeline for tag pushes (#888) * Add auto-release pipeline for tag pushes * Document major version RPA update commit: cef8d8ee0881e8bb5c4e3add1f515c7c63823b28 author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi8/ubi-minimal:latest docker digest to e2e7f26 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 06ea5a54b525e5e8e6c80fd9265c81e4df4f107b author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi9 docker tag to v9.8-1789348643 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 70ba84a6746d4e3a9811ddc79aa3c66cc8c2167e author: Josh Branham Harden update and revert framework test commit: c5f8cc34310af499077e38cab92c0c6f75a67edd author: Josh Branham Fix image tag test for detached HEAD commit: 267668233eddfcc29f72421db66045127e67dd27 author: Josh Branham Resolve boilerplate image tag from published images --- boilerplate/_data/backing-image-tag | 2 +- boilerplate/_data/last-boilerplate-commit | 2 +- boilerplate/_lib/common.sh | 4 +- .../openshift/golang-osd-e2e/e2e-template.yml | 9 + .../openshift/golang-osd-e2e/e2e_client.go | 194 ++++++++++++++++++ .../gangway-bridge-template.yml | 30 ++- boilerplate/openshift/golang-osd-e2e/update | 13 +- boilerplate/update | 9 +- test/e2e/Dockerfile | 5 +- test/e2e/README.md | 4 +- test/e2e/e2e-template.yml | 9 + test/e2e/e2e_client.go | 194 ++++++++++++++++++ test/e2e/gangway-bridge-template.yml | 30 ++- 13 files changed, 471 insertions(+), 34 deletions(-) create mode 100644 boilerplate/openshift/golang-osd-e2e/e2e_client.go create mode 100644 test/e2e/e2e_client.go diff --git a/boilerplate/_data/backing-image-tag b/boilerplate/_data/backing-image-tag index 284dd19e..a0f9a4b4 100644 --- a/boilerplate/_data/backing-image-tag +++ b/boilerplate/_data/backing-image-tag @@ -1 +1 @@ -image-v8.4.3 +latest diff --git a/boilerplate/_data/last-boilerplate-commit b/boilerplate/_data/last-boilerplate-commit index db9c1a3d..43a25524 100644 --- a/boilerplate/_data/last-boilerplate-commit +++ b/boilerplate/_data/last-boilerplate-commit @@ -1 +1 @@ -195c29d18279f69ea5b4e53ae9d90a795c27144a +1191e12968520575f7177f1ff055e966b91cd52a diff --git a/boilerplate/_lib/common.sh b/boilerplate/_lib/common.sh index a3298af5..7063bf6e 100755 --- a/boilerplate/_lib/common.sh +++ b/boilerplate/_lib/common.sh @@ -74,7 +74,7 @@ current_branch() { ) } -## image_exits_in_repo IMAGE_URI +## image_exists_in_repo IMAGE_URI # # Checks whether IMAGE_URI -- e.g. quay.io/app-sre/osd-metrics-exporter:abcd123 # -- exists in the remote repository. @@ -190,7 +190,7 @@ IMAGE_NAMESPACE=openshift IMAGE_NAME=boilerplate # LATEST_IMAGE_TAG may be set manually or by `update`, in which case # that's the value we want to use. -if [[ -z "$LATEST_IMAGE_TAG" ]]; then +if [[ -z "$LATEST_IMAGE_TAG" && -z "$SKIP_LATEST_IMAGE_TAG_RESOLUTION" ]]; then # (Non-ancient) consumers will have the tag in this file. if [[ -f ${CONVENTION_ROOT}/_data/backing-image-tag ]]; then LATEST_IMAGE_TAG=$(cat ${CONVENTION_ROOT}/_data/backing-image-tag) diff --git a/boilerplate/openshift/golang-osd-e2e/e2e-template.yml b/boilerplate/openshift/golang-osd-e2e/e2e-template.yml index 3ec951bd..acf1c260 100644 --- a/boilerplate/openshift/golang-osd-e2e/e2e-template.yml +++ b/boilerplate/openshift/golang-osd-e2e/e2e-template.yml @@ -52,6 +52,10 @@ objects: secret: secretName: osde2e-gcp-credentials optional: true + - name: test-run-results + emptyDir: {} + - name: tmp + emptyDir: {} containers: - name: osde2e image: quay.io/redhat-services-prod/osde2e-cicada-tenant/osde2e:latest @@ -80,6 +84,10 @@ objects: - name: osde2e-gcp-sc readOnly: true mountPath: "/etc/osde2e-gcp-sc" + - name: test-run-results + mountPath: /test-run-results + - name: tmp + mountPath: /tmp resources: requests: cpu: "300m" @@ -89,6 +97,7 @@ objects: memory: "1200Mi" securityContext: runAsNonRoot: true + readOnlyRootFilesystem: true allowPrivilegeEscalation: false capabilities: drop: [ "ALL" ] diff --git a/boilerplate/openshift/golang-osd-e2e/e2e_client.go b/boilerplate/openshift/golang-osd-e2e/e2e_client.go new file mode 100644 index 00000000..a291f260 --- /dev/null +++ b/boilerplate/openshift/golang-osd-e2e/e2e_client.go @@ -0,0 +1,194 @@ +// THIS FILE IS GENERATED BY BOILERPLATE. DO NOT EDIT. +//go:build osde2e +// +build osde2e + +package osde2etests + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + openshiftapi "github.com/openshift/api" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// E2EClientOption configures an E2EClient during construction. +// Use WithScheme to register operator-specific CRD types. +type E2EClientOption func(*e2eClientConfig) error + +// e2eClientConfig holds options accumulated before client creation. +type e2eClientConfig struct { + extraSchemes []func(*runtime.Scheme) error +} + +// WithScheme returns an option that registers additional types with the +// client's runtime.Scheme. Each operator calls this to add its own CRD +// types without editing this boilerplate file. +// +// Example: +// +// c, err := NewE2EClient(log, WithScheme(myoperatorv1.AddToScheme)) +func WithScheme(addToScheme func(*runtime.Scheme) error) E2EClientOption { + return func(cfg *e2eClientConfig) error { + cfg.extraSchemes = append(cfg.extraSchemes, addToScheme) + return nil + } +} + +// E2EClient wraps controller-runtime's client.Client with convenience +// methods for OSD operator e2e tests. It replaces the osde2e-common +// dependency entirely. +type E2EClient struct { + client.Client + config *rest.Config + log logr.Logger + extraSchemes []func(*runtime.Scheme) error +} + +// NewE2EClient creates an E2EClient by loading kubeconfig from the +// KUBECONFIG env var (falling back to ~/.kube/config). Pass WithScheme +// options to register operator-specific CRD types. +func NewE2EClient(log logr.Logger, opts ...E2EClientOption) (*E2EClient, error) { + cfg, err := loadKubeConfig() + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %w", err) + } + return NewE2EClientFromConfig(cfg, log, opts...) +} + +// NewE2EClientFromConfig creates an E2EClient from an explicit +// rest.Config. The base scheme always includes core/v1, apps/v1, and +// the OpenShift API types. Pass WithScheme options to register +// additional types (e.g. operator CRDs). +func NewE2EClientFromConfig(cfg *rest.Config, log logr.Logger, opts ...E2EClientOption) (*E2EClient, error) { + var ecfg e2eClientConfig + for _, opt := range opts { + if err := opt(&ecfg); err != nil { + return nil, fmt.Errorf("failed to apply client option: %w", err) + } + } + + scheme := runtime.NewScheme() + + // Register base types that every OSD operator test needs. + if err := corev1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to register core/v1: %w", err) + } + if err := appsv1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to register apps/v1: %w", err) + } + if err := openshiftapi.Install(scheme); err != nil { + return nil, fmt.Errorf("failed to register openshift api: %w", err) + } + + // Register operator-specific types supplied via WithScheme. + for _, addToScheme := range ecfg.extraSchemes { + if err := addToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to register extra scheme: %w", err) + } + } + + c, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("failed to create controller-runtime client: %w", err) + } + return &E2EClient{Client: c, config: cfg, log: log, extraSchemes: ecfg.extraSchemes}, nil +} + +// Get wraps client.Get with positional name/namespace args for API +// compatibility with the old osde2e-common openshift.Client. +func (c *E2EClient) Get(ctx context.Context, name, namespace string, obj client.Object) error { + return c.Client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, obj) +} + +// GetScheme returns the client's runtime.Scheme. +func (c *E2EClient) GetScheme() *runtime.Scheme { + return c.Client.Scheme() +} + +// GetConfig returns the rest.Config used by this client. +func (c *E2EClient) GetConfig() *rest.Config { + return c.config +} + +// Impersonate returns a new E2EClient that acts as the given user and +// groups. The "system:authenticated" and "system:authenticated:oauth" +// groups are added automatically when user is non-empty. +func (c *E2EClient) Impersonate(user string, groups ...string) (*E2EClient, error) { + if user != "" { + groups = append(groups, "system:authenticated", "system:authenticated:oauth") + } + impersonatedCfg := rest.CopyConfig(c.config) + impersonatedCfg.Impersonate = rest.ImpersonationConfig{UserName: user, Groups: groups} + + // Preserve operator-specific schemes so the impersonated client + // can still work with custom CRD types. + opts := make([]E2EClientOption, len(c.extraSchemes)) + for i, s := range c.extraSchemes { + opts[i] = WithScheme(s) + } + return NewE2EClientFromConfig(impersonatedCfg, c.log, opts...) +} + +const ( + metadataConfigMap = "osd-cluster-metadata" + configNamespace = "openshift-config" +) + +// getClusterMetadata reads the osd-cluster-metadata configmap from the +// openshift-config namespace. +func (c *E2EClient) getClusterMetadata(ctx context.Context) (map[string]string, error) { + var cm corev1.ConfigMap + if err := c.Get(ctx, metadataConfigMap, configNamespace, &cm); err != nil { + return nil, err + } + return cm.Data, nil +} + +// IsSTS returns true if the cluster is configured with STS (Security +// Token Service) authentication. +func (c *E2EClient) IsSTS(ctx context.Context) (bool, error) { + data, err := c.getClusterMetadata(ctx) + if err != nil { + return false, err + } + return data["api.openshift.com_sts"] == "true", nil +} + +// GetProvider returns the cloud provider name from cluster metadata +// (e.g. "aws", "gcp"). +func (c *E2EClient) GetProvider(ctx context.Context) (string, error) { + data, err := c.getClusterMetadata(ctx) + if err != nil { + return "", err + } + return data["hive.openshift.io_cluster-platform"], nil +} + +// GetRegion returns the cloud region from cluster metadata (e.g. +// "us-east-1"). +func (c *E2EClient) GetRegion(ctx context.Context) (string, error) { + data, err := c.getClusterMetadata(ctx) + if err != nil { + return "", err + } + return data["hive.openshift.io_cluster-region"], nil +} + +// loadKubeConfig loads a rest.Config using client-go's default loading +// rules. This properly handles multi-path KUBECONFIG (e.g. +// "/a/config:/b/config") and falls back to ~/.kube/config when the env +// var is unset. +func loadKubeConfig() (*rest.Config, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + return kubeConfig.ClientConfig() +} diff --git a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml index d1a925a4..544aab99 100644 --- a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml +++ b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml @@ -58,13 +58,13 @@ objects: [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } [[ "${INITIAL_DELAY}" =~ ^[0-9]+$ ]] || { log "ERROR: INITIAL_DELAY must be a non-negative integer"; exit 1; } - if [[ "${INITIAL_DELAY}" -gt 0 ]]; then - log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." - sleep "${INITIAL_DELAY}" - fi - # Backoff sum: base 30s doubling each retry, capped at 900s, plus 15s max jitter - MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) + MAX_BACKOFF_SUM=0 + for (( i = 0; i < MAX_RETRIES; i++ )); do + b=$(( i < 5 ? 30 * (1 << i) : 900 )) + (( b > 900 )) && b=900 + MAX_BACKOFF_SUM=$(( MAX_BACKOFF_SUM + b + 15 )) + done # Each attempt may overshoot TIMEOUT by up to max(POLL_INTERVAL, 300s max backoff) + # status-request max-time (30s) on the last poll cycle POLL_OVERSHOOT=$(( (POLL_INTERVAL > 300 ? POLL_INTERVAL : 300) + 30 )) @@ -77,9 +77,14 @@ objects: exit 1 fi + if [[ "${INITIAL_DELAY}" -gt 0 ]]; then + log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." + sleep "${INITIAL_DELAY}" + fi + BODY='{"job_execution_type":"1"}' if [[ -n "${JOB_ENVS:-}" ]]; then - ENVS=$(echo "${JOB_ENVS}" | jq -Rn '[inputs // input | split(",")[] | split("=") | {(.[0]): .[1:] | join("=")}] | add' <<< "${JOB_ENVS}") + ENVS=$(echo "${JOB_ENVS}" | jq -Rn '[inputs | split(",") | .[] | select(length > 0) | split("=") | {(.[0]): (.[1:] | join("="))}] | add // empty') BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') fi @@ -150,15 +155,20 @@ objects: esac done log "Prow logs: ${PROW_URL}" - log "Timeout"; return 1 + log "Timeout"; return 2 } ATTEMPT=0 while true; do ATTEMPT=$((ATTEMPT + 1)) log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" - if trigger_and_poll; then + rc=0 + trigger_and_poll || rc=$? + if [[ $rc -eq 0 ]]; then exit 0 + elif [[ $rc -eq 2 ]]; then + log "Timed out waiting for Prow job — not retrying (the job may still be running)" + exit 1 fi if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then log "All attempts exhausted" @@ -168,7 +178,7 @@ objects: log "Skipping backoff (already waited for Retry-After)" RATE_LIMITED_WAITED=0 else - BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) + BACKOFF=$(( ATTEMPT > 6 ? 900 : 30 * (1 << (ATTEMPT - 1)) )) [[ $BACKOFF -gt 900 ]] && BACKOFF=900 JITTER=$(( RANDOM % 16 )) DELAY=$(( BACKOFF + JITTER )) diff --git a/boilerplate/openshift/golang-osd-e2e/update b/boilerplate/openshift/golang-osd-e2e/update index 4e3a06aa..34d41c29 100755 --- a/boilerplate/openshift/golang-osd-e2e/update +++ b/boilerplate/openshift/golang-osd-e2e/update @@ -34,6 +34,10 @@ OPERATOR_NAME_CAMEL_CASE=${OPERATOR_PROPER_NAME// /} mkdir -p "${E2E_SUITE_DIRECTORY}" +# Copy the generalized e2e client helper (always overwritten). +echo "syncing ${E2E_SUITE_DIRECTORY}/e2e_client.go" +cp "$(dirname $0)/e2e_client.go" "${E2E_SUITE_DIRECTORY}/e2e_client.go" + E2E_SUITE_BUILDER_IMAGE=registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.26-openshift-4.22 if [[ -n ${KONFLUX_BUILDS} ]]; then E2E_SUITE_BUILDER_IMAGE="brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_9_1.26" @@ -44,11 +48,14 @@ tee "${E2E_SUITE_DIRECTORY}/Dockerfile" < /(path-to)/kubeconfig +(umask 077 && ocm get /api/clusters_mgmt/v1/clusters/(cluster-id)/credentials | jq -r .kubeconfig > /(path-to)/kubeconfig) 5. Run test suite using diff --git a/boilerplate/update b/boilerplate/update index c3f0cc40..8bc6f78f 100755 --- a/boilerplate/update +++ b/boilerplate/update @@ -160,10 +160,11 @@ if [ ! -f "$CONFIG_FILE" ]; then exit 1 fi -# The most recent build image tag. Export this for individual `update` scripts. -if [[ -z "$LATEST_IMAGE_TAG" ]]; then - export LATEST_IMAGE_TAG=$(cd $BP_CLONE; git describe --tags --abbrev=0 --match image-v*) -fi +# Consuming repositories use the stable `latest` tag for the boilerplate +# backing image. Keep an explicit value for local testing or pinning, but do +# not infer a tag from the consumer's old backing-image-tag file or from Git +# release tags. This also migrates existing consumers to `latest` below. +export LATEST_IMAGE_TAG="${LATEST_IMAGE_TAG:-latest}" # The boilerplate commit hash. Export for convention `update` scripts. export BOILERPLATE_COMMIT=$(cd ${BP_CLONE} && git rev-parse HEAD) diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile index b83a20cc..39c7021e 100644 --- a/test/e2e/Dockerfile +++ b/test/e2e/Dockerfile @@ -1,9 +1,12 @@ # THIS FILE IS GENERATED BY BOILERPLATE. DO NOT EDIT. FROM brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_9_1.26 as builder WORKDIR /go/src/github.com/openshift/managed-cluster-validating-webhooks/ +COPY go.mod go.sum ./ +RUN go mod download COPY . . RUN CGO_ENABLED=0 GOFLAGS="-mod=mod" go test ./test/e2e -v -c --tags=osde2e -o /e2e.test FROM registry.access.redhat.com/ubi8/ubi-minimal:latest -COPY --from=builder ./e2e.test e2e.test +COPY --from=builder /e2e.test /e2e.test +USER 1001 ENTRYPOINT [ "/e2e.test" ] diff --git a/test/e2e/README.md b/test/e2e/README.md index 16231b89..21a16ff0 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -3,10 +3,10 @@ When updating your operator, add e2e tests for new functionality and ensure exis 1. Run "make e2e-binary-build" to make sure e2e tests build 2. Deploy your new version of operator in a test cluster -3. Run "go install github.com/onsi/ginkgo/ginkgo@latest" +3. Run "go install github.com/onsi/ginkgo/v2/ginkgo@latest" 4. Get kubeadmin credentials from your cluster using -ocm get /api/clusters_mgmt/v1/clusters/(cluster-id)/credentials | jq -r .kubeconfig > /(path-to)/kubeconfig +(umask 077 && ocm get /api/clusters_mgmt/v1/clusters/(cluster-id)/credentials | jq -r .kubeconfig > /(path-to)/kubeconfig) 5. Run test suite using diff --git a/test/e2e/e2e-template.yml b/test/e2e/e2e-template.yml index 1a6121ad..e4152e1f 100644 --- a/test/e2e/e2e-template.yml +++ b/test/e2e/e2e-template.yml @@ -52,6 +52,10 @@ objects: secret: secretName: osde2e-gcp-credentials optional: true + - name: test-run-results + emptyDir: {} + - name: tmp + emptyDir: {} containers: - name: osde2e image: quay.io/redhat-services-prod/osde2e-cicada-tenant/osde2e:latest @@ -80,6 +84,10 @@ objects: - name: osde2e-gcp-sc readOnly: true mountPath: "/etc/osde2e-gcp-sc" + - name: test-run-results + mountPath: /test-run-results + - name: tmp + mountPath: /tmp resources: requests: cpu: "300m" @@ -89,6 +97,7 @@ objects: memory: "1200Mi" securityContext: runAsNonRoot: true + readOnlyRootFilesystem: true allowPrivilegeEscalation: false capabilities: drop: [ "ALL" ] diff --git a/test/e2e/e2e_client.go b/test/e2e/e2e_client.go new file mode 100644 index 00000000..a291f260 --- /dev/null +++ b/test/e2e/e2e_client.go @@ -0,0 +1,194 @@ +// THIS FILE IS GENERATED BY BOILERPLATE. DO NOT EDIT. +//go:build osde2e +// +build osde2e + +package osde2etests + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + openshiftapi "github.com/openshift/api" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// E2EClientOption configures an E2EClient during construction. +// Use WithScheme to register operator-specific CRD types. +type E2EClientOption func(*e2eClientConfig) error + +// e2eClientConfig holds options accumulated before client creation. +type e2eClientConfig struct { + extraSchemes []func(*runtime.Scheme) error +} + +// WithScheme returns an option that registers additional types with the +// client's runtime.Scheme. Each operator calls this to add its own CRD +// types without editing this boilerplate file. +// +// Example: +// +// c, err := NewE2EClient(log, WithScheme(myoperatorv1.AddToScheme)) +func WithScheme(addToScheme func(*runtime.Scheme) error) E2EClientOption { + return func(cfg *e2eClientConfig) error { + cfg.extraSchemes = append(cfg.extraSchemes, addToScheme) + return nil + } +} + +// E2EClient wraps controller-runtime's client.Client with convenience +// methods for OSD operator e2e tests. It replaces the osde2e-common +// dependency entirely. +type E2EClient struct { + client.Client + config *rest.Config + log logr.Logger + extraSchemes []func(*runtime.Scheme) error +} + +// NewE2EClient creates an E2EClient by loading kubeconfig from the +// KUBECONFIG env var (falling back to ~/.kube/config). Pass WithScheme +// options to register operator-specific CRD types. +func NewE2EClient(log logr.Logger, opts ...E2EClientOption) (*E2EClient, error) { + cfg, err := loadKubeConfig() + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %w", err) + } + return NewE2EClientFromConfig(cfg, log, opts...) +} + +// NewE2EClientFromConfig creates an E2EClient from an explicit +// rest.Config. The base scheme always includes core/v1, apps/v1, and +// the OpenShift API types. Pass WithScheme options to register +// additional types (e.g. operator CRDs). +func NewE2EClientFromConfig(cfg *rest.Config, log logr.Logger, opts ...E2EClientOption) (*E2EClient, error) { + var ecfg e2eClientConfig + for _, opt := range opts { + if err := opt(&ecfg); err != nil { + return nil, fmt.Errorf("failed to apply client option: %w", err) + } + } + + scheme := runtime.NewScheme() + + // Register base types that every OSD operator test needs. + if err := corev1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to register core/v1: %w", err) + } + if err := appsv1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to register apps/v1: %w", err) + } + if err := openshiftapi.Install(scheme); err != nil { + return nil, fmt.Errorf("failed to register openshift api: %w", err) + } + + // Register operator-specific types supplied via WithScheme. + for _, addToScheme := range ecfg.extraSchemes { + if err := addToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to register extra scheme: %w", err) + } + } + + c, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("failed to create controller-runtime client: %w", err) + } + return &E2EClient{Client: c, config: cfg, log: log, extraSchemes: ecfg.extraSchemes}, nil +} + +// Get wraps client.Get with positional name/namespace args for API +// compatibility with the old osde2e-common openshift.Client. +func (c *E2EClient) Get(ctx context.Context, name, namespace string, obj client.Object) error { + return c.Client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, obj) +} + +// GetScheme returns the client's runtime.Scheme. +func (c *E2EClient) GetScheme() *runtime.Scheme { + return c.Client.Scheme() +} + +// GetConfig returns the rest.Config used by this client. +func (c *E2EClient) GetConfig() *rest.Config { + return c.config +} + +// Impersonate returns a new E2EClient that acts as the given user and +// groups. The "system:authenticated" and "system:authenticated:oauth" +// groups are added automatically when user is non-empty. +func (c *E2EClient) Impersonate(user string, groups ...string) (*E2EClient, error) { + if user != "" { + groups = append(groups, "system:authenticated", "system:authenticated:oauth") + } + impersonatedCfg := rest.CopyConfig(c.config) + impersonatedCfg.Impersonate = rest.ImpersonationConfig{UserName: user, Groups: groups} + + // Preserve operator-specific schemes so the impersonated client + // can still work with custom CRD types. + opts := make([]E2EClientOption, len(c.extraSchemes)) + for i, s := range c.extraSchemes { + opts[i] = WithScheme(s) + } + return NewE2EClientFromConfig(impersonatedCfg, c.log, opts...) +} + +const ( + metadataConfigMap = "osd-cluster-metadata" + configNamespace = "openshift-config" +) + +// getClusterMetadata reads the osd-cluster-metadata configmap from the +// openshift-config namespace. +func (c *E2EClient) getClusterMetadata(ctx context.Context) (map[string]string, error) { + var cm corev1.ConfigMap + if err := c.Get(ctx, metadataConfigMap, configNamespace, &cm); err != nil { + return nil, err + } + return cm.Data, nil +} + +// IsSTS returns true if the cluster is configured with STS (Security +// Token Service) authentication. +func (c *E2EClient) IsSTS(ctx context.Context) (bool, error) { + data, err := c.getClusterMetadata(ctx) + if err != nil { + return false, err + } + return data["api.openshift.com_sts"] == "true", nil +} + +// GetProvider returns the cloud provider name from cluster metadata +// (e.g. "aws", "gcp"). +func (c *E2EClient) GetProvider(ctx context.Context) (string, error) { + data, err := c.getClusterMetadata(ctx) + if err != nil { + return "", err + } + return data["hive.openshift.io_cluster-platform"], nil +} + +// GetRegion returns the cloud region from cluster metadata (e.g. +// "us-east-1"). +func (c *E2EClient) GetRegion(ctx context.Context) (string, error) { + data, err := c.getClusterMetadata(ctx) + if err != nil { + return "", err + } + return data["hive.openshift.io_cluster-region"], nil +} + +// loadKubeConfig loads a rest.Config using client-go's default loading +// rules. This properly handles multi-path KUBECONFIG (e.g. +// "/a/config:/b/config") and falls back to ~/.kube/config when the env +// var is unset. +func loadKubeConfig() (*rest.Config, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + return kubeConfig.ClientConfig() +} diff --git a/test/e2e/gangway-bridge-template.yml b/test/e2e/gangway-bridge-template.yml index d1a925a4..544aab99 100644 --- a/test/e2e/gangway-bridge-template.yml +++ b/test/e2e/gangway-bridge-template.yml @@ -58,13 +58,13 @@ objects: [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } [[ "${INITIAL_DELAY}" =~ ^[0-9]+$ ]] || { log "ERROR: INITIAL_DELAY must be a non-negative integer"; exit 1; } - if [[ "${INITIAL_DELAY}" -gt 0 ]]; then - log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." - sleep "${INITIAL_DELAY}" - fi - # Backoff sum: base 30s doubling each retry, capped at 900s, plus 15s max jitter - MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) + MAX_BACKOFF_SUM=0 + for (( i = 0; i < MAX_RETRIES; i++ )); do + b=$(( i < 5 ? 30 * (1 << i) : 900 )) + (( b > 900 )) && b=900 + MAX_BACKOFF_SUM=$(( MAX_BACKOFF_SUM + b + 15 )) + done # Each attempt may overshoot TIMEOUT by up to max(POLL_INTERVAL, 300s max backoff) + # status-request max-time (30s) on the last poll cycle POLL_OVERSHOOT=$(( (POLL_INTERVAL > 300 ? POLL_INTERVAL : 300) + 30 )) @@ -77,9 +77,14 @@ objects: exit 1 fi + if [[ "${INITIAL_DELAY}" -gt 0 ]]; then + log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." + sleep "${INITIAL_DELAY}" + fi + BODY='{"job_execution_type":"1"}' if [[ -n "${JOB_ENVS:-}" ]]; then - ENVS=$(echo "${JOB_ENVS}" | jq -Rn '[inputs // input | split(",")[] | split("=") | {(.[0]): .[1:] | join("=")}] | add' <<< "${JOB_ENVS}") + ENVS=$(echo "${JOB_ENVS}" | jq -Rn '[inputs | split(",") | .[] | select(length > 0) | split("=") | {(.[0]): (.[1:] | join("="))}] | add // empty') BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') fi @@ -150,15 +155,20 @@ objects: esac done log "Prow logs: ${PROW_URL}" - log "Timeout"; return 1 + log "Timeout"; return 2 } ATTEMPT=0 while true; do ATTEMPT=$((ATTEMPT + 1)) log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" - if trigger_and_poll; then + rc=0 + trigger_and_poll || rc=$? + if [[ $rc -eq 0 ]]; then exit 0 + elif [[ $rc -eq 2 ]]; then + log "Timed out waiting for Prow job — not retrying (the job may still be running)" + exit 1 fi if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then log "All attempts exhausted" @@ -168,7 +178,7 @@ objects: log "Skipping backoff (already waited for Retry-After)" RATE_LIMITED_WAITED=0 else - BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) + BACKOFF=$(( ATTEMPT > 6 ? 900 : 30 * (1 << (ATTEMPT - 1)) )) [[ $BACKOFF -gt 900 ]] && BACKOFF=900 JITTER=$(( RANDOM % 16 )) DELAY=$(( BACKOFF + JITTER ))