From 77680e40d5b85a2efc51a699413ed9e943a0d5a4 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 19:31:40 -0700
Subject: [PATCH 1/9] CI: Install automatic retry for dependency downloads

---
 .github/workflows/build.yml | 44 +++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fa0b773f..5ee98e18 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -123,6 +123,7 @@ jobs:
       CC: ${{ matrix.compiler }}
       CFLAGS: ${{ matrix.extra_flags }}
       CONFIGURE_ARGS: --disable-mpi-compat --enable-rpath ${{ inputs.configure_args }}
+      CURL_OPTIONS: --retry 10 --retry-all-errors --fail -L
       GASNET_BACKTRACE: 1
       GASNET_SPAWN_VERBOSE: 1
       GASNET_SPAWNFN: L
@@ -135,20 +136,41 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v6
 
+    - name: Install shell functions
+      run: |
+        mkdir -p ~/.bash_helpers
+        cat << 'EOF' > ~/.bash_helpers/custom_functions.sh
+        retry() {
+          local retry_cnt=0
+          local retry_delay
+          while ! "$@" ; do
+            retry_cnt=$(expr $retry_cnt + 1)
+            if [[ $retry_cnt > 5 ]] ; then exit 1 ; fi
+            ( set +x ;
+              echo "::warning::Command failed: <$@> Pausing before retry $retry_cnt..." );
+            retry_delay=$(( 5 ** retry_cnt ))
+            if [[ $retry_delay > 600 ]] ; then retry_delay=600 ; fi
+            sleep $retry_delay
+          done
+        }
+        EOF
+        # Register file in BASH_ENV so it automatically loads in all future steps
+        echo "BASH_ENV=~/.bash_helpers/custom_functions.sh" >> $GITHUB_ENV
+
     - name: Install Ubuntu Native Dependencies
       if: ${{ contains(matrix.os, 'ubuntu') && matrix.container == '' && matrix.compiler == 'gcc' }}
       run: |
-        sudo apt update
+        retry sudo apt update
         #sudo apt list -a 'gcc-*'
-        sudo apt install -y build-essential
+        retry sudo apt install -y build-essential
         if [[ ${COMPILER_VERSION} -lt 15 ]] ; then
-          sudo apt install -y gcc-${COMPILER_VERSION}
+          retry sudo apt install -y gcc-${COMPILER_VERSION}
         else
-          curl -L https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh
+          curl $CURL_OPTIONS https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh
           chmod +x install-homebrew.sh
           env CI=1 ./install-homebrew.sh
           HOMEBREW_PREFIX="/home/linuxbrew/.linuxbrew"
-          ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils
+          retry ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils
           ls -al ${HOMEBREW_PREFIX}/bin
           echo "PATH=${HOMEBREW_PREFIX}/bin:${PATH}" >> "$GITHUB_ENV"
           : Homebrew GCC@15 needs binutils 2.44+
@@ -162,26 +184,26 @@ jobs:
     - name: Install Ubuntu Container Dependencies
       if: ${{ contains(matrix.os, 'ubuntu') && matrix.container != '' && !contains(matrix.container, 'phhargrove') }}
       run: |
-        apt update
-        apt install -y build-essential # pkg-config make git curl
+        retry apt update
+        retry apt install -y build-essential # pkg-config make git curl
 
     - name: Install macOS Dependencies
       if: contains(matrix.os, 'macos')
       run: |
         # silence Homebrew tap trust warnings:
         (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
-        brew update
+        retry brew update
         if [[ "${{ matrix.compiler }}@${{ matrix.version }}" == gcc@12 ]] ; then
-          brew install gcc@12
+          retry brew install gcc@12
         fi
         if [[ ${{ matrix.compiler }} == 'gcc' && ${COMPILER_VERSION} -gt 15 ]] ; then
-          brew install gcc@${COMPILER_VERSION}
+          retry brew install gcc@${COMPILER_VERSION}
         fi
 
     - name: Install LLVM on macOS
       if: contains(matrix.os, 'macos') && matrix.compiler == 'clang'
       run: |
-        brew install llvm@${COMPILER_VERSION} 
+        retry brew install llvm@${COMPILER_VERSION} 
         # Prepend homebrew clang to PATH:
         echo "PATH=$(brew --prefix)/opt/llvm/bin:${PATH}" >> "$GITHUB_ENV"
 

From d6f12385afec2963caec07be141aad757bb3d335 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 19:38:11 -0700
Subject: [PATCH 2/9] CI: Filter harmless nuisance warnings from Homebrew

---
 .github/workflows/build.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5ee98e18..52417ec5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -124,6 +124,7 @@ jobs:
       CFLAGS: ${{ matrix.extra_flags }}
       CONFIGURE_ARGS: --disable-mpi-compat --enable-rpath ${{ inputs.configure_args }}
       CURL_OPTIONS: --retry 10 --retry-all-errors --fail -L
+      BREW_WARNING_FILTER: 's/^[[:punct:]]*warning[[:punct:]]*(You are using macOS)/warn: \1/i'
       GASNET_BACKTRACE: 1
       GASNET_SPAWN_VERBOSE: 1
       GASNET_SPAWNFN: L
@@ -194,16 +195,19 @@ jobs:
         (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
         retry brew update
         if [[ "${{ matrix.compiler }}@${{ matrix.version }}" == gcc@12 ]] ; then
-          retry brew install gcc@12
+          retry brew install gcc@12 \
+              2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         fi
         if [[ ${{ matrix.compiler }} == 'gcc' && ${COMPILER_VERSION} -gt 15 ]] ; then
-          retry brew install gcc@${COMPILER_VERSION}
+          retry brew install gcc@${COMPILER_VERSION} \
+              2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         fi
 
     - name: Install LLVM on macOS
       if: contains(matrix.os, 'macos') && matrix.compiler == 'clang'
       run: |
-        retry brew install llvm@${COMPILER_VERSION} 
+        retry brew install llvm@${COMPILER_VERSION} \
+              2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         # Prepend homebrew clang to PATH:
         echo "PATH=$(brew --prefix)/opt/llvm/bin:${PATH}" >> "$GITHUB_ENV"
 

From 5889fe51ff68ee5656d3052b0590854bbc602a4f Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 19:44:42 -0700
Subject: [PATCH 3/9] CI: Disable gcc@16 on macOS+Intel

---
 .github/workflows/build.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 52417ec5..5a0e6373 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -49,6 +49,14 @@ jobs:
         version: [ 12, 13, 14, 15, 16 ]
         extra_flags: [ -g ]
 
+        exclude:
+          
+          # Homebrew removed Intel bottles for gcc@16
+          - os: macos-15-intel
+            version: 16
+          - os: macos-26-intel
+            version: 16
+
         include:
           - os: ubuntu-22.04
             compiler: gcc

From 3f6b7378f0e6aac19802e1a32f64ca71dc1ef1c6 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 19:47:01 -0700
Subject: [PATCH 4/9] CI: Update LLVM clang configs via brew

---
 .github/workflows/build.yml | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5a0e6373..468b8989 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -64,21 +64,20 @@ jobs:
 
           # --- LLVM clang coverage ---
 
-          - os: macos-14
-            compiler: clang
-            version: 22
-          - os: macos-15
-            compiler: clang
-            version: 22
-          - os: macos-15-intel
-            compiler: clang
-            version: 22
-          - os: macos-26
-            compiler: clang
-            version: 22
-          - os: macos-26-intel
-            compiler: clang
-            version: 22
+          # Build currently broken with clang 23.1.2
+          #- os: macos-15
+          #  compiler: clang
+          #  version: 23
+          #- os: macos-26
+          #  compiler: clang
+          #  version: 23
+          # No bottles for mac-intel
+          #- os: macos-15-intel
+          #  compiler: clang
+          #  version: 22
+          #- os: macos-26-intel
+          #  compiler: clang
+          #  version: 22
 
           # https://hub.docker.com/r/phhargrove/llvm-flang/tags
           - os: ubuntu-24.04

From a9896e711696d4672fb88cdf3878c89fb8839d3b Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 19:48:42 -0700
Subject: [PATCH 5/9] CI: Fix macOS Homebrew Configuration

---
 .github/workflows/build.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 468b8989..825b16a2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -195,6 +195,20 @@ jobs:
         retry apt update
         retry apt install -y build-essential # pkg-config make git curl
 
+    - name: Fix macOS Homebrew Configuration
+      if: ${{ contains(matrix.os, 'macos') }}
+      run: |
+        # silence Homebrew tap trust warnings:
+        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
+        # set some Homebrew knobs to accelerate installation
+        # but only when we are not simulating the user environment brew-via-install workflow
+        if [[ -z "${{ matrix.brew_via_install }}" ]] ; then
+          echo HOMEBREW_NO_ANALYTICS=1 >> "$GITHUB_ENV"
+          echo HOMEBREW_NO_AUTO_UPDATE=1 >> "$GITHUB_ENV"
+          echo HOMEBREW_NO_GITHUB_API=1 >> "$GITHUB_ENV"
+          echo HOMEBREW_NO_INSTALL_CLEANUP=1 >> "$GITHUB_ENV"
+        fi
+
     - name: Install macOS Dependencies
       if: contains(matrix.os, 'macos')
       run: |

From 936728851400dc2dbddb49448dbbb74bdb9aa036 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 19:49:47 -0700
Subject: [PATCH 6/9] CI: Add ubuntu-26.04-arm coverage

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 825b16a2..2869f4fb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -44,7 +44,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ macos-14, macos-15, macos-15-intel, macos-26, macos-26-intel, ubuntu-24.04 ]
+        os: [ macos-14, macos-15, macos-15-intel, macos-26, macos-26-intel, ubuntu-24.04, ubuntu-26.04-arm ]
         compiler: [ gcc ]
         version: [ 12, 13, 14, 15, 16 ]
         extra_flags: [ -g ]

From a4ddcc4fe6f2bc234dc1d01c2cb5f440ed14fe29 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 20:27:23 -0700
Subject: [PATCH 7/9] CI: Ensure selected g++ version is installed on Linux

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2869f4fb..b75748b4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -171,8 +171,8 @@ jobs:
         retry sudo apt update
         #sudo apt list -a 'gcc-*'
         retry sudo apt install -y build-essential
-        if [[ ${COMPILER_VERSION} -lt 15 ]] ; then
-          retry sudo apt install -y gcc-${COMPILER_VERSION}
+        if [[ ${COMPILER_VERSION} -lt 15 ]] || type -p gcc-${COMPILER_VERSION} ; then
+          retry sudo apt install -y gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION}
         else
           curl $CURL_OPTIONS https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh
           chmod +x install-homebrew.sh

From fd254bbe5797204362b8006cd777355d6f717bd4 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 21:41:12 -0700
Subject: [PATCH 8/9] CI: Tweak brew tap adjustment, to ignore intermittent
 harmless errors

---
 .github/workflows/build.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b75748b4..368b3042 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -199,7 +199,7 @@ jobs:
       if: ${{ contains(matrix.os, 'macos') }}
       run: |
         # silence Homebrew tap trust warnings:
-        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
+        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/^::*//;s/^##*//' ; brew trust azure/bicep ; echo )
         # set some Homebrew knobs to accelerate installation
         # but only when we are not simulating the user environment brew-via-install workflow
         if [[ -z "${{ matrix.brew_via_install }}" ]] ; then
@@ -212,8 +212,6 @@ jobs:
     - name: Install macOS Dependencies
       if: contains(matrix.os, 'macos')
       run: |
-        # silence Homebrew tap trust warnings:
-        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
         retry brew update
         if [[ "${{ matrix.compiler }}@${{ matrix.version }}" == gcc@12 ]] ; then
           retry brew install gcc@12 \

From de3460e21faddbb3d44716d22e5f3247d0442a65 Mon Sep 17 00:00:00 2001
From: Dan Bonachea <dobonachea@lbl.gov>
Date: Sun, 27 Sep 2026 22:42:39 -0700
Subject: [PATCH 9/9] CI: Add coverage for Apple Clang

---
 .github/workflows/build.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 368b3042..c7112642 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -62,6 +62,21 @@ jobs:
             compiler: gcc
             version: 12 # no package available for gcc 13+
 
+          # --- Apple clang coverage ---
+
+          - os: macos-15
+            compiler: apple-clang
+            version: 17
+          - os: macos-15-intel
+            compiler: apple-clang
+            version: 17
+          - os: macos-26
+            compiler: apple-clang
+            version: 21
+          - os: macos-26-intel
+            compiler: apple-clang
+            version: 21
+
           # --- LLVM clang coverage ---
 
           # Build currently broken with clang 23.1.2
@@ -127,7 +142,7 @@ jobs:
 
     env:
       COMPILER_VERSION: ${{ matrix.version }}
-      CC: ${{ matrix.compiler }}
+      CC: ${{ matrix.compiler == 'apple-clang' && 'clang' || matrix.compiler }}
       CFLAGS: ${{ matrix.extra_flags }}
       CONFIGURE_ARGS: --disable-mpi-compat --enable-rpath ${{ inputs.configure_args }}
       CURL_OPTIONS: --retry 10 --retry-all-errors --fail -L