diff --git a/bin/installAll.sh b/bin/installAll.sh index 5691113..6dc96eb 100755 --- a/bin/installAll.sh +++ b/bin/installAll.sh @@ -168,6 +168,35 @@ STARTUP_WAVES=( "mc-application-manager mc-workflow-manager mc-cost-optimizer-fe" ) +# Starts every wave detached (-d), then one final full-stack `run -d` for the +# services no wave entry point reaches (cost-optimizer collectors/rightsizers, +# cb-mapui, ...). Waves must always be detached: an attached `compose up` never +# returns, so later waves would never start. +run_startup_waves() { + local wave_num=0 + local run_exit + for wave_services in "${STARTUP_WAVES[@]}"; do + wave_num=$((wave_num + 1)) + echo "" + echo "---- Wave $wave_num/${#STARTUP_WAVES[@]}: $wave_services ----" + ./mcc infra run -d -s "$wave_services" + run_exit=$? + if [ $run_exit -ne 0 ]; then + report_run_failure "$run_exit" "Wave $wave_num ($wave_services)" + exit 1 + fi + done + + echo "" + echo "---- Remaining services (not reached by any wave) ----" + ./mcc infra run -d + run_exit=$? + if [ $run_exit -ne 0 ]; then + report_run_failure "$run_exit" "remaining services" + exit 1 + fi +} + # Prints a consistent failure banner for a failed `./mcc infra run` invocation. report_run_failure() { local exit_code="$1" @@ -543,20 +572,14 @@ case $RUN_MODE in exit 1 fi - wave_num=0 - for wave_services in "${STARTUP_WAVES[@]}"; do - wave_num=$((wave_num + 1)) - echo "" - echo "---- Wave $wave_num/${#STARTUP_WAVES[@]}: $wave_services ----" - ./mcc infra run -s "$wave_services" - run_exit=$? - if [ $run_exit -ne 0 ]; then - report_run_failure "$run_exit" "Wave $wave_num ($wave_services)" - exit 1 - fi - done - + run_startup_waves check_post_initial + + # Attach to the whole stack only after every wave is up (Ctrl+C to detach; + # containers keep running) + echo "" + echo "All services started. Attaching to logs (Ctrl+C to stop following)..." + ./mcc infra run ;; background) echo "" @@ -579,18 +602,7 @@ case $RUN_MODE in echo "Image download and initial setup in progress..." echo "" - wave_num=0 - for wave_services in "${STARTUP_WAVES[@]}"; do - wave_num=$((wave_num + 1)) - echo "" - echo "---- Wave $wave_num/${#STARTUP_WAVES[@]}: $wave_services ----" - ./mcc infra run -d -s "$wave_services" - run_exit=$? - if [ $run_exit -ne 0 ]; then - report_run_failure "$run_exit" "Wave $wave_num ($wave_services)" - exit 1 - fi - done + run_startup_waves echo "" echo "Image download and initial setup completed." diff --git a/conf/docker/.env.setup b/conf/docker/.env.setup index 9166e21..434c5b3 100644 --- a/conf/docker/.env.setup +++ b/conf/docker/.env.setup @@ -159,7 +159,7 @@ MC_IAM_MANAGER_DEFAULT_WORKSPACE_NAME=ws01 MC_IAM_MANAGER_AWS_STS_ENDPOINT=https://sts.amazonaws.com MC_IAM_MANAGER_AWS_ACCOUNT_ID=notyet MC_IAM_MANAGER_AWS_IDENTITY_PROVIDER_ARN=arn:aws:iam::${MC_IAM_MANAGER_AWS_ACCOUNT_ID}:oidc-provider/${MC_IAM_MANAGER_KEYCLOAK_DOMAIN}/realms/${MC_IAM_MANAGER_KEYCLOAK_OIDC_CLIENT_NAME} -MC_IAM_MANAGER_AWS_IDENTITY_ROLE_ARN=arn:aws:iam::${MC_IAM_MANAGER_KEYCLOAK_DOMAIN}:role/mciam-platformadmin +MC_IAM_MANAGER_AWS_IDENTITY_ROLE_ARN=arn:aws:iam::${MC_IAM_MANAGER_AWS_ACCOUNT_ID}:role/mciam-platformadmin MC_IAM_MANAGER_CSP_ROLE_PREFIX=mciam diff --git a/conf/docker/docker-compose.yaml b/conf/docker/docker-compose.yaml index a898ece..3f4fac0 100644 --- a/conf/docker/docker-compose.yaml +++ b/conf/docker/docker-compose.yaml @@ -1042,7 +1042,10 @@ services: curl -fsS -u "$${JENKINS_USERNAME}:$${JENKINS_PASSWORD}" http://localhost:8080/api/json > /dev/null <<: *default-health-check - start_period: 6m + # First boot downloads ~100 plugins and restarts Jenkins; a clean install + # was observed taking ~9.5 min. start_period only suppresses failures, so a + # generous value costs nothing once Jenkins is actually ready. + start_period: 15m mc-workflow-manager: diff --git a/conf/docker/tool/init.groovy.d/basic-security.groovy b/conf/docker/tool/init.groovy.d/basic-security.groovy index 2c8560e..cb6523a 100644 --- a/conf/docker/tool/init.groovy.d/basic-security.groovy +++ b/conf/docker/tool/init.groovy.d/basic-security.groovy @@ -97,6 +97,37 @@ plugins.each { pluginName -> } } +// deploy() also queues each dependency as its own InstallationJob, and a failed +// download (e.g. "Connection reset" from the update site) does not throw -- it +// only leaves that job in Failure state. Without a retry the missing dependency +// (e.g. joda-time-api) makes dependents fail to load after restart, and Jenkins +// only converges after several restart cycles. +def failedInstalls = { + def latestJobs = [:] + uc.getJobs().findAll { it instanceof UpdateCenter.InstallationJob }.each { job -> + latestJobs[job.plugin.name] = job + } + latestJobs.findAll { name, job -> job.status instanceof UpdateCenter.DownloadJob.Failure }.keySet() +} + +def maxInstallRetries = 5 +for (int attempt = 1; attempt <= maxInstallRetries; attempt++) { + def failed = failedInstalls() + if (failed.isEmpty()) { + break + } + println "--> Retrying failed plugin downloads (${attempt}/${maxInstallRetries}): ${failed.join(', ')}" + sleep(5000L * attempt) + failed.each { pluginName -> + uc.getPlugin(pluginName)?.deploy()?.get() + } +} + +def stillFailed = failedInstalls() +if (!stillFailed.isEmpty()) { + throw new IllegalStateException("Jenkins plugin download failed after ${maxInstallRetries} retries: ${stillFailed.join(', ')}") +} + println "--> Saving Jenkins state..." instance.save()