From 4dd08cc88b013e20c67a72e5ac0d4dd9e14f623e Mon Sep 17 00:00:00 2001 From: Predrag Knezevic Date: Fri, 31 Jul 2026 18:57:35 +0200 Subject: [PATCH] fix: make demo e2e catalog queries resilient to transient failures The generate-demos CI job fails ~45% of the time with jq exit status 5 on the ClusterCatalog Quickstart scenario. Three issues contribute: - jq -s (slurp mode) buffers the entire operatorhubio FBC response in memory before processing, risking system errors on large catalogs - catalog content queries run exactly once with no retry, so any transient port-forward or network hiccup fails the step immediately - bash() does not attach stderr to ExitError, making failures opaque Remove jq slurp mode so each JSON object is processed in constant memory. Wrap CatalogContainsSomePackages, PackageHasSomeChannels, and PackageHasSomeBundles in waitFor for retry on transient errors. Add curl --retry for transport-level resilience. Inject stderr into ExitError in bash() to match k8sClient diagnostics. Co-Authored-By: Claude --- test/e2e/steps/demo_steps.go | 95 +++++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/test/e2e/steps/demo_steps.go b/test/e2e/steps/demo_steps.go index d4ac355b1a..2a07d29549 100644 --- a/test/e2e/steps/demo_steps.go +++ b/test/e2e/steps/demo_steps.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -41,6 +42,10 @@ func bash(ctx context.Context, script string) (string, error) { if err != nil { logger.V(1).Info("Failed to run", "command", script, "stderr", stderr, "error", err) + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitErr.Stderr = stderrBuf.Bytes() + } } logger.V(1).Info("Output", "command", script, "output", stdout) @@ -81,7 +86,15 @@ func CatalogReportsConditionWithoutReason(ctx context.Context, catalogUserName, func ensureCatalogPortForward(ctx context.Context) (string, error) { sc := scenarioCtx(ctx) if sc.catalogAddr != "" { - return sc.catalogAddr, nil + if catalogPortForwardAlive(sc.catalogAddr) { + return sc.catalogAddr, nil + } + logger.V(1).Info("Catalog port-forward is dead, re-establishing", "addr", sc.catalogAddr) + if sc.catalogCleanup != nil { + sc.catalogCleanup() + } + sc.catalogAddr = "" + sc.catalogCleanup = nil } addr, cleanup, err := portForward(ctx, componentNamespaces["catalogd"], "service/catalogd-service", 443) @@ -92,68 +105,72 @@ func ensureCatalogPortForward(ctx context.Context) (string, error) { sc.catalogCleanup = cleanup waitFor(ctx, func() bool { - client := &http.Client{ - Timeout: 3 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec - DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, - }, - } - resp, err := client.Get(fmt.Sprintf("https://%s/", addr)) - if err != nil { - return false - } - resp.Body.Close() - return true + return catalogPortForwardAlive(addr) }) return addr, nil } +func catalogPortForwardAlive(addr string) bool { + client := &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, + }, + } + resp, err := client.Get(fmt.Sprintf("https://%s/", addr)) + if err != nil { + return false + } + resp.Body.Close() + return true +} + func catalogCurlJq(ctx context.Context, catalogName, jqFilter string) (string, error) { addr, err := ensureCatalogPortForward(ctx) if err != nil { return "", err } script := fmt.Sprintf( - `curl -s -k https://%s/catalogs/%s/api/v1/all | jq -s '%s'`, + `curl -sS -k --retry 3 --retry-delay 2 --retry-all-errors https://%s/catalogs/%s/api/v1/all | jq '%s'`, addr, catalogName, jqFilter, ) return bash(ctx, script) } func CatalogContainsSomePackages(ctx context.Context, catalogName string) error { - out, err := catalogCurlJq(ctx, catalogName, - `.[] | select(.schema == "olm.package") | .name`) - if err != nil { - return err - } - if strings.TrimSpace(out) == "" { - return fmt.Errorf("catalog %q contains no packages", catalogName) - } + waitFor(ctx, func() bool { + out, err := catalogCurlJq(ctx, catalogName, + `select(.schema == "olm.package") | .name`) + if err != nil { + return false + } + return strings.TrimSpace(out) != "" + }) return nil } func PackageHasSomeChannels(ctx context.Context, packageName, catalogName string) error { - out, err := catalogCurlJq(ctx, catalogName, - fmt.Sprintf(`.[] | select(.schema == "olm.channel") | select(.package == "%s") | .name`, packageName)) - if err != nil { - return err - } - if strings.TrimSpace(out) == "" { - return fmt.Errorf("package %q in catalog %q has no channels", packageName, catalogName) - } + waitFor(ctx, func() bool { + out, err := catalogCurlJq(ctx, catalogName, + fmt.Sprintf(`select(.schema == "olm.channel") | select(.package == "%s") | .name`, packageName)) + if err != nil { + return false + } + return strings.TrimSpace(out) != "" + }) return nil } func PackageHasSomeBundles(ctx context.Context, packageName, catalogName string) error { - out, err := catalogCurlJq(ctx, catalogName, - fmt.Sprintf(`.[] | select(.schema == "olm.bundle") | select(.package == "%s") | .name`, packageName)) - if err != nil { - return err - } - if strings.TrimSpace(out) == "" { - return fmt.Errorf("package %q in catalog %q has no bundles", packageName, catalogName) - } + waitFor(ctx, func() bool { + out, err := catalogCurlJq(ctx, catalogName, + fmt.Sprintf(`select(.schema == "olm.bundle") | select(.package == "%s") | .name`, packageName)) + if err != nil { + return false + } + return strings.TrimSpace(out) != "" + }) return nil }