[NO REVIEW]: CI Updates

.github/workflows/build.yml

@@ -44,33 +44,55 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ macos-14, macos-15, macos-15-intel, macos-26, macos-26-intel, ubuntu-24.04 ]
+        os: [ macos-14, macos-15, macos-15-intel, macos-26, macos-26-intel, ubuntu-24.04, ubuntu-26.04-arm ]
         compiler: [ gcc ]
         version: [ 12, 13, 14, 15, 16 ]
         extra_flags: [ -g ]
 
+        exclude:
+
+          # Homebrew removed Intel bottles for gcc@16
+          - os: macos-15-intel
+            version: 16
+          - os: macos-26-intel
+            version: 16
+
         include:
           - os: ubuntu-22.04
             compiler: gcc
             version: 12 # no package available for gcc 13+
 
-          # --- LLVM clang coverage ---
+          # --- Apple clang coverage ---
 
-          - os: macos-14
-            compiler: clang
-            version: 22
           - os: macos-15
-            compiler: clang
-            version: 22
+            compiler: apple-clang
+            version: 17
           - os: macos-15-intel
-            compiler: clang
-            version: 22
+            compiler: apple-clang
+            version: 17
           - os: macos-26
-            compiler: clang
-            version: 22
+            compiler: apple-clang
+            version: 21
           - os: macos-26-intel
-            compiler: clang
-            version: 22
+            compiler: apple-clang
+            version: 21
+
+          # --- LLVM clang coverage ---
+
+          # Build currently broken with clang 23.1.2
+          #- os: macos-15
+          #  compiler: clang
+          #  version: 23
+          #- os: macos-26
+          #  compiler: clang
+          #  version: 23
+          # No bottles for mac-intel
+          #- os: macos-15-intel
+          #  compiler: clang
+          #  version: 22
+          #- os: macos-26-intel
+          #  compiler: clang
+          #  version: 22
 
           # https://hub.docker.com/r/phhargrove/llvm-flang/tags
           - os: ubuntu-24.04
@@ -120,9 +142,11 @@ jobs:
 
     env:
       COMPILER_VERSION: ${{ matrix.version }}
-      CC: ${{ matrix.compiler }}
+      CC: ${{ matrix.compiler == 'apple-clang' && 'clang' || matrix.compiler }}
       CFLAGS: ${{ matrix.extra_flags }}
       CONFIGURE_ARGS: --disable-mpi-compat --enable-rpath ${{ inputs.configure_args }}
+      CURL_OPTIONS: --retry 10 --retry-all-errors --fail -L
+      BREW_WARNING_FILTER: 's/^[[:punct:]]*warning[[:punct:]]*(You are using macOS)/warn: \1/i'
       GASNET_BACKTRACE: 1
       GASNET_SPAWN_VERBOSE: 1
       GASNET_SPAWNFN: L
@@ -135,20 +159,41 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v6
 
+    - name: Install shell functions
+      run: |
+        mkdir -p ~/.bash_helpers
+        cat << 'EOF' > ~/.bash_helpers/custom_functions.sh
+        retry() {
+          local retry_cnt=0
+          local retry_delay
+          while ! "$@" ; do
+            retry_cnt=$(expr $retry_cnt + 1)
+            if [[ $retry_cnt > 5 ]] ; then exit 1 ; fi
+            ( set +x ;
+              echo "::warning::Command failed: <$@> Pausing before retry $retry_cnt..." );
+            retry_delay=$(( 5 ** retry_cnt ))
+            if [[ $retry_delay > 600 ]] ; then retry_delay=600 ; fi
+            sleep $retry_delay
+          done
+        }
+        EOF
+        # Register file in BASH_ENV so it automatically loads in all future steps
+        echo "BASH_ENV=~/.bash_helpers/custom_functions.sh" >> $GITHUB_ENV
+
     - name: Install Ubuntu Native Dependencies
       if: ${{ contains(matrix.os, 'ubuntu') && matrix.container == '' && matrix.compiler == 'gcc' }}
       run: |
-        sudo apt update
+        retry sudo apt update
         #sudo apt list -a 'gcc-*'
-        sudo apt install -y build-essential
-        if [[ ${COMPILER_VERSION} -lt 15 ]] ; then
-          sudo apt install -y gcc-${COMPILER_VERSION}
+        retry sudo apt install -y build-essential
+        if [[ ${COMPILER_VERSION} -lt 15 ]] || type -p gcc-${COMPILER_VERSION} ; then
+          retry sudo apt install -y gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION}
         else
-          curl -L https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh
+          curl $CURL_OPTIONS https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh
           chmod +x install-homebrew.sh
           env CI=1 ./install-homebrew.sh
           HOMEBREW_PREFIX="/home/linuxbrew/.linuxbrew"
-          ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils
+          retry ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils
           ls -al ${HOMEBREW_PREFIX}/bin
           echo "PATH=${HOMEBREW_PREFIX}/bin:${PATH}" >> "$GITHUB_ENV"
           : Homebrew GCC@15 needs binutils 2.44+
@@ -162,26 +207,41 @@ jobs:
     - name: Install Ubuntu Container Dependencies
       if: ${{ contains(matrix.os, 'ubuntu') && matrix.container != '' && !contains(matrix.container, 'phhargrove') }}
       run: |
-        apt update
-        apt install -y build-essential # pkg-config make git curl
+        retry apt update
+        retry apt install -y build-essential # pkg-config make git curl
+
+    - name: Fix macOS Homebrew Configuration
+      if: ${{ contains(matrix.os, 'macos') }}
+      run: |
+        # silence Homebrew tap trust warnings:
+        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/^::*//;s/^##*//' ; brew trust azure/bicep ; echo )
+        # set some Homebrew knobs to accelerate installation
+        # but only when we are not simulating the user environment brew-via-install workflow
+        if [[ -z "${{ matrix.brew_via_install }}" ]] ; then
+          echo HOMEBREW_NO_ANALYTICS=1 >> "$GITHUB_ENV"
+          echo HOMEBREW_NO_AUTO_UPDATE=1 >> "$GITHUB_ENV"
+          echo HOMEBREW_NO_GITHUB_API=1 >> "$GITHUB_ENV"
+          echo HOMEBREW_NO_INSTALL_CLEANUP=1 >> "$GITHUB_ENV"
+        fi
 
     - name: Install macOS Dependencies
       if: contains(matrix.os, 'macos')
       run: |
-        # silence Homebrew tap trust warnings:
-        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
-        brew update
+        retry brew update
         if [[ "${{ matrix.compiler }}@${{ matrix.version }}" == gcc@12 ]] ; then
-          brew install gcc@12
+          retry brew install gcc@12 \
+              2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         fi
         if [[ ${{ matrix.compiler }} == 'gcc' && ${COMPILER_VERSION} -gt 15 ]] ; then
-          brew install gcc@${COMPILER_VERSION}
+          retry brew install gcc@${COMPILER_VERSION} \
+              2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         fi
 
     - name: Install LLVM on macOS
       if: contains(matrix.os, 'macos') && matrix.compiler == 'clang'
       run: |
-        brew install llvm@${COMPILER_VERSION} 
+        retry brew install llvm@${COMPILER_VERSION} \
+              2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         # Prepend homebrew clang to PATH:
         echo "PATH=$(brew --prefix)/opt/llvm/bin:${PATH}" >> "$GITHUB_ENV"